@scrypted/server 0.123.33 → 0.123.35

package/src/scrypted-server-main.ts

@@ -1,6 +1,6 @@
 import bodyParser from 'body-parser';
 import cookieParser from 'cookie-parser';
-import crypto from 'crypto';
+import crypto, { scrypt } from 'crypto';
 import { once } from 'events';
 import express, { Request } from 'express';
 import fs from 'fs';
@@ -23,7 +23,7 @@ import { getNpmPackageInfo } from './services/plugin';
 import { setScryptedUserPassword, UsersService } from './services/users';
 import { sleep } from './sleep';
 import { ONE_DAY_MILLISECONDS, UserToken } from './usertoken';
-import { createClusterServer } from './scrypted-cluster-main';
+import { createClusterServer, startClusterClient } from './scrypted-cluster-main';
 import { getScryptedClusterMode } from './cluster/cluster-setup';
 
 export type Runtime = ScryptedRuntime;
@@ -45,10 +45,11 @@ installSourceMapSupport({
 });
 
 let workerInspectPort: number = undefined;
+let workerInspectAddress: string = undefined;
 
 async function doconnect(): Promise<net.Socket> {
     return new Promise((resolve, reject) => {
-        const target = net.connect(workerInspectPort, '127.0.0.1');
+        const target = net.connect(workerInspectPort, workerInspectAddress);
         target.once('error', reject)
         target.once('connect', () => resolve(target))
     })
@@ -100,7 +101,7 @@ app.use(bodyParser.raw({ type: 'application/*', limit: 100000000 }) as any)
 async function start(mainFilename: string, options?: {
     onRuntimeCreated?: (runtime: ScryptedRuntime) => Promise<void>,
 }) {
-    console.log('Cluster server starting.');
+    console.log('Scrypted server starting.');
     const volumeDir = getScryptedVolume();
     await fs.promises.mkdir(volumeDir, {
         recursive: true
@@ -350,6 +351,14 @@ async function start(mainFilename: string, options?: {
 
     const scrypted = new ScryptedRuntime(mainFilename, db, insecure, secure, app);
     await options?.onRuntimeCreated?.(scrypted);
+
+    const clusterMode = getScryptedClusterMode();
+    if (clusterMode?.[0] === 'server') {
+        console.log('Cluster server starting.');
+        const clusterServer = createClusterServer(mainFilename, scrypted, keyPair);
+        await listenServerPort('SCRYPTED_CLUSTER_SERVER', clusterMode[2], clusterServer);
+    }
+
     await scrypted.start();
 
 
@@ -465,11 +474,20 @@ async function start(mainFilename: string, options?: {
         waitDebug.catch(() => { });
 
         workerInspectPort = Math.round(Math.random() * 10000) + 30000;
+        workerInspectAddress = '127.0.0.1';
         try {
-            await scrypted.installPlugin(plugin, {
+            const host = await scrypted.installPlugin(plugin, {
                 waitDebug,
                 inspectPort: workerInspectPort,
             });
+
+            const clusterWorkerId = await host.clusterWorkerId;
+            if (clusterWorkerId) {
+                const clusterWorker = scrypted.clusterWorkers.get(clusterWorkerId);
+                if (clusterWorker) {
+                    workerInspectAddress = clusterWorker.address;
+                }
+            }
         }
         catch (e) {
             res.header('Content-Type', 'text/plain');
@@ -480,6 +498,7 @@ async function start(mainFilename: string, options?: {
 
         res.send({
             workerInspectPort,
+            workerInspectAddress,
         });
     });
 
@@ -725,12 +744,6 @@ async function start(mainFilename: string, options?: {
     await listenServerPort('SCRYPTED_SECURE_PORT', SCRYPTED_SECURE_PORT, secure);
     await listenServerPort('SCRYPTED_INSECURE_PORT', SCRYPTED_INSECURE_PORT, insecure);
 
-    const clusterMode = getScryptedClusterMode();
-    if (clusterMode?.[0] === 'server') {
-        const clusterServer = createClusterServer(scrypted, keyPair);
-        await listenServerPort('SCRYPTED_CLUSTER_SERVER', clusterMode[2], clusterServer);
-    }
-
     console.log('#######################################################');
     console.log(`Scrypted Volume           : ${volumeDir}`);
     console.log(`Scrypted Server (Local)   : https://localhost:${SCRYPTED_SECURE_PORT}/`);

package/src/services/cluster-fork.ts

@@ -1,14 +1,41 @@
-import { ClusterWorker } from "@scrypted/types";
 import { matchesClusterLabels } from "../cluster/cluster-labels";
-import type { ScryptedRuntime } from "../runtime";
-import type { ClusterForkOptions, ClusterForkParam, PeerLiveness, RunningClusterWorker } from "../scrypted-cluster-main";
 import type { RuntimeWorkerOptions } from "../plugin/runtime/runtime-worker";
+import { RpcPeer } from "../rpc";
+import type { ScryptedRuntime } from "../runtime";
+import type { ClusterForkOptions, ClusterForkParam, ClusterForkResultInterface, PeerLiveness, RunningClusterWorker } from "../scrypted-cluster-main";
+
+class WrappedForkResult implements ClusterForkResultInterface {
+    [RpcPeer.PROPERTY_PROXY_PROPERTIES] = {
+        clusterWorkerId: undefined as string,
+    };
+
+    constructor(public clusterWorkerId: string, public forkResult: Promise<ClusterForkResultInterface>) {
+        this[RpcPeer.PROPERTY_PROXY_PROPERTIES].clusterWorkerId = clusterWorkerId;
+    }
+
+    async kill() {
+        const fr = await this.forkResult.catch(() => { });
+        if (!fr)
+            return;
+        await fr.kill();
+    }
+
+    async getResult() {
+        const fr = await this.forkResult;
+        return fr.getResult();
+    }
+
+    async waitKilled() {
+        const fr = await this.forkResult;
+        await fr.waitKilled();
+    }
+}
 
 export class ClusterForkService {
     constructor(public runtime: ScryptedRuntime) { }
 
     async fork(runtimeWorkerOptions: RuntimeWorkerOptions, options: ClusterForkOptions, peerLiveness: PeerLiveness, getZip: () => Promise<Buffer>) {
-        const matchingWorkers = [...this.runtime.clusterWorkers.entries()].map(([id, worker]) => ({
+        let matchingWorkers = [...this.runtime.clusterWorkers.entries()].map(([id, worker]) => ({
             worker,
             matches: matchesClusterLabels(options, worker.labels),
         }))
@@ -17,7 +44,6 @@ export class ClusterForkService {
                 // and worker id must match if provided
                 return matches && (!options.clusterWorkerId || worker.id === options.clusterWorkerId);
             });
-        matchingWorkers.sort((a, b) => b.worker.labels.length - a.worker.labels.length);
 
         let worker: RunningClusterWorker;
 
@@ -26,32 +52,52 @@ export class ClusterForkService {
         if (options.id)
             worker = matchingWorkers.find(({ worker }) => [...worker.forks].find(f => f.id === options.id))?.worker;
 
-        // TODO: round robin?
-        worker ||= matchingWorkers[0]?.worker;
-
         if (!worker) {
-            if (options.clusterWorkerId)
-                throw new Error(`no worker found for cluster id ${options.clusterWorkerId}`);
-            throw new Error(`no worker found for cluster labels ${JSON.stringify(options.labels)}`);
+            // sort by number of matches, to find the best match.
+            matchingWorkers.sort((a, b) => b.matches - a.matches);
+
+            const bestMatch = matchingWorkers[0];
+
+            if (!bestMatch) {
+                if (options.clusterWorkerId)
+                    throw new Error(`no worker found for cluster id ${options.clusterWorkerId}`);
+                throw new Error(`no worker found for cluster labels ${JSON.stringify(options.labels)}`);
+            }
+
+            // filter out workers that are not equivalent to the best match.
+            // this enforces the "prefer" label.
+            matchingWorkers = matchingWorkers.filter(({ matches }) => matches === bestMatch.matches)
+                // sort by number of forks, to distribute load.
+                .sort((a, b) => a.worker.forks.size - b.worker.forks.size);
+
+            worker = matchingWorkers[0]?.worker;
         }
 
-        const fork: ClusterForkParam = await worker.peer.getParam('fork');
-        const forkResult = await fork(options.runtime, runtimeWorkerOptions, peerLiveness, getZip);
+        console.log('forking to worker', worker.id, options);
+
+        worker.fork ||= worker.peer.getParam('fork');
+        const fork: ClusterForkParam = await worker.fork;
+        const forkResultPromise = fork(options.runtime, runtimeWorkerOptions, peerLiveness, getZip);
+
         options.id ||= this.runtime.findPluginDevice(runtimeWorkerOptions.packageJson.name)?._id;
         worker.forks.add(options);
-        forkResult.waitKilled().catch(() => { }).finally(() => {
-            worker.forks.delete(options);
+
+        forkResultPromise.then(forkResult => {
+            forkResult.clusterWorkerId = worker.id;
+            forkResult.waitKilled().catch(() => { }).finally(() => {
+                worker.forks.delete(options);
+            })
         });
 
-        forkResult.clusterWorkerId = worker.id;
-        return forkResult;
+        const ret: ClusterForkResultInterface = new WrappedForkResult(worker.id, forkResultPromise);
+        return ret;
     };
 
     async getClusterWorkers() {
         const ret: any = {};
         for (const worker of this.runtime.clusterWorkers.values()) {
             ret[worker.id] = {
-                name: worker.peer.peerName,
+                name: worker.name,
                 labels: worker.labels,
                 forks: [...worker.forks],
             };