@scrypted/server 0.123.34 → 0.123.35

package/python/rpc_reader.py

@@ -16,7 +16,7 @@ import json
 
 class BufferSerializer(rpc.RpcSerializer):
     def serialize(self, value, serializationContext):
-        return base64.b64encode(value).decode('utf8')
+        return base64.b64encode(value).decode("utf8")
 
     def deserialize(self, value, serializationContext):
         return base64.b64decode(value)
@@ -24,15 +24,15 @@ class BufferSerializer(rpc.RpcSerializer):
 
 class SidebandBufferSerializer(rpc.RpcSerializer):
     def serialize(self, value, serializationContext):
-        buffers = serializationContext.get('buffers', None)
+        buffers = serializationContext.get("buffers", None)
         if not buffers:
             buffers = []
-            serializationContext['buffers'] = buffers
+            serializationContext["buffers"] = buffers
         buffers.append(value)
         return len(buffers) - 1
 
     def deserialize(self, value, serializationContext):
-        buffers: List = serializationContext.get('buffers', None)
+        buffers: List = serializationContext.get("buffers", None)
         buffer = buffers.pop()
         return buffer
 
@@ -56,7 +56,7 @@ class RpcFileTransport(RpcTransport):
         super().__init__()
         self.readFd = readFd
         self.writeFd = writeFd
-        self.executor = ThreadPoolExecutor(1, 'rpc-read')
+        self.executor = ThreadPoolExecutor(1, "rpc-read")
 
     def osReadExact(self, size: int):
         b = bytes(0)
@@ -64,7 +64,7 @@ class RpcFileTransport(RpcTransport):
             got = os.read(self.readFd, size)
             if not len(got):
                 self.executor.shutdown(False)
-                raise Exception('rpc end of stream reached')
+                raise Exception("rpc end of stream reached")
             size -= len(got)
             b += got
         return b
@@ -73,7 +73,7 @@ class RpcFileTransport(RpcTransport):
         lengthBytes = self.osReadExact(4)
         typeBytes = self.osReadExact(1)
         type = typeBytes[0]
-        length = int.from_bytes(lengthBytes, 'big')
+        length = int.from_bytes(lengthBytes, "big")
         data = self.osReadExact(length - 1)
         if type == 1:
             return data
@@ -81,11 +81,13 @@ class RpcFileTransport(RpcTransport):
         return message
 
     async def read(self):
-        return await asyncio.get_event_loop().run_in_executor(self.executor, lambda: self.readMessageInternal())
+        return await asyncio.get_event_loop().run_in_executor(
+            self.executor, lambda: self.readMessageInternal()
+        )
 
     def writeMessage(self, type: int, buffer, reject):
         length = len(buffer) + 1
-        lb = length.to_bytes(4, 'big')
+        lb = length.to_bytes(4, "big")
         try:
             for b in [lb, bytes([type]), buffer]:
                 os.write(self.writeFd, b)
@@ -94,14 +96,18 @@ class RpcFileTransport(RpcTransport):
                 reject(e)
 
     def writeJSON(self, j, reject):
-        return self.writeMessage(0, bytes(json.dumps(j, allow_nan=False), 'utf8'), reject)
+        return self.writeMessage(
+            0, bytes(json.dumps(j, allow_nan=False), "utf8"), reject
+        )
 
     def writeBuffer(self, buffer, reject):
         return self.writeMessage(1, buffer, reject)
 
 
 class RpcStreamTransport(RpcTransport):
-    def __init__(self, reader: asyncio.StreamReader, writer: asyncio.StreamWriter) -> None:
+    def __init__(
+        self, reader: asyncio.StreamReader, writer: asyncio.StreamWriter
+    ) -> None:
         super().__init__()
         self.reader = reader
         self.writer = writer
@@ -110,7 +116,7 @@ class RpcStreamTransport(RpcTransport):
         lengthBytes = await self.reader.readexactly(4)
         typeBytes = await self.reader.readexactly(1)
         type = typeBytes[0]
-        length = int.from_bytes(lengthBytes, 'big')
+        length = int.from_bytes(lengthBytes, "big")
         data = await self.reader.readexactly(length - 1)
         if type == 1:
             return data
@@ -119,7 +125,7 @@ class RpcStreamTransport(RpcTransport):
 
     def writeMessage(self, type: int, buffer, reject):
         length = len(buffer) + 1
-        lb = length.to_bytes(4, 'big')
+        lb = length.to_bytes(4, "big")
         try:
             for b in [lb, bytes([type]), buffer]:
                 self.writer.write(b)
@@ -128,7 +134,9 @@ class RpcStreamTransport(RpcTransport):
                 reject(e)
 
     def writeJSON(self, j, reject):
-        return self.writeMessage(0, bytes(json.dumps(j, allow_nan=False), 'utf8'), reject)
+        return self.writeMessage(
+            0, bytes(json.dumps(j, allow_nan=False), "utf8"), reject
+        )
 
     def writeBuffer(self, buffer, reject):
         return self.writeMessage(1, buffer, reject)
@@ -141,7 +149,9 @@ class RpcConnectionTransport(RpcTransport):
         self.executor = concurrent.futures.ThreadPoolExecutor(max_workers=1)
 
     async def read(self):
-        return await asyncio.get_event_loop().run_in_executor(self.executor, lambda: self.connection.recv())
+        return await asyncio.get_event_loop().run_in_executor(
+            self.executor, lambda: self.connection.recv()
+        )
 
     def writeMessage(self, json, reject):
         try:
@@ -158,23 +168,20 @@ class RpcConnectionTransport(RpcTransport):
 
 
 async def readLoop(loop, peer: rpc.RpcPeer, rpcTransport: RpcTransport):
-    deserializationContext = {
-        'buffers': []
-    }
+    deserializationContext = {"buffers": []}
 
     while True:
         message = await rpcTransport.read()
 
         if type(message) != dict:
-            deserializationContext['buffers'].append(message)
+            deserializationContext["buffers"].append(message)
             continue
 
         asyncio.run_coroutine_threadsafe(
-            peer.handleMessage(message, deserializationContext), loop)
+            peer.handleMessage(message, deserializationContext), loop
+        )
 
-        deserializationContext = {
-            'buffers': []
-        }
+        deserializationContext = {"buffers": []}
 
 
 async def prepare_peer_readloop(loop: AbstractEventLoop, rpcTransport: RpcTransport):
@@ -185,7 +192,7 @@ async def prepare_peer_readloop(loop: AbstractEventLoop, rpcTransport: RpcTransp
     def send(message, reject=None, serializationContext=None):
         with mutex:
             if serializationContext:
-                buffers = serializationContext.get('buffers', None)
+                buffers = serializationContext.get("buffers", None)
                 if buffers:
                     for buffer in buffers:
                         rpcTransport.writeBuffer(buffer, reject)
@@ -193,10 +200,10 @@ async def prepare_peer_readloop(loop: AbstractEventLoop, rpcTransport: RpcTransp
             rpcTransport.writeJSON(message, reject)
 
     peer = rpc.RpcPeer(send)
-    peer.nameDeserializerMap['Buffer'] = SidebandBufferSerializer()
-    peer.constructorSerializerMap[bytes] = 'Buffer'
-    peer.constructorSerializerMap[bytearray] = 'Buffer'
-    peer.constructorSerializerMap[memoryview] = 'Buffer'
+    peer.nameDeserializerMap["Buffer"] = SidebandBufferSerializer()
+    peer.constructorSerializerMap[bytes] = "Buffer"
+    peer.constructorSerializerMap[bytearray] = "Buffer"
+    peer.constructorSerializerMap[memoryview] = "Buffer"
 
     async def peerReadLoop():
         try:

package/src/cluster/cluster-labels.ts

@@ -9,13 +9,19 @@ export function matchesClusterLabels(options: ClusterForkOptions, labels: string
     }
 
     // if there is nothing in the any list, consider it matched
-    let foundAny = !options?.labels?.any?.length;
-    for (const label of options.labels?.any || []) {
-        if (labels.includes(label)) {
-            matched++;
-            foundAny = true;
+    let foundAny: boolean;
+    if (options?.labels?.any?.length) {
+        for (const label of options.labels.any) {
+            if (labels.includes(label)) {
+                foundAny = true;
+                break;
+            }
         }
     }
+    else {
+        foundAny = true;
+    }
+
     if (!foundAny)
         return 0;
 
@@ -40,3 +46,8 @@ export function needsClusterForkWorker(options: ClusterForkOptions) {
         && options
         && (!matchesClusterLabels(options, getClusterLabels()) || options.clusterWorkerId);
 }
+
+export function utilizesClusterForkWorker(options: ClusterForkOptions) {
+    return process.env.SCRYPTED_CLUSTER_ADDRESS
+        && (options?.labels || options?.clusterWorkerId);
+}

package/src/cluster/cluster-setup.ts

@@ -249,8 +249,15 @@ export function setupCluster(peer: RpcPeer) {
         if (address === SCRYPTED_CLUSTER_ADDRESS && proxyId.startsWith('n-')) {
             const parts = proxyId.split('-');
             const pid = parseInt(parts[1]);
-            if (pid === process.pid)
-                return connectIPCObject(clusterObject, parseInt(parts[2]));
+            const tid = parseInt(parts[2]);
+            if (pid === process.pid) {
+                if (worker_threads.isMainThread && tid === worker_threads.threadId) {
+                    // main thread can't call itself, so this may be a different thread cluster.
+                }
+                else {
+                    return connectIPCObject(clusterObject, parseInt(parts[2]));
+                }
+            }
         }
 
         try {

package/src/plugin/plugin-host.ts

@@ -4,7 +4,7 @@ import * as io from 'engine.io';
 import fs from 'fs';
 import os from 'os';
 import WebSocket from 'ws';
-import { needsClusterForkWorker } from '../cluster/cluster-labels';
+import { utilizesClusterForkWorker } from '../cluster/cluster-labels';
 import { setupCluster } from '../cluster/cluster-setup';
 import { Plugin } from '../db-types';
 import { IOServer, IOServerSocket } from '../io';
@@ -351,7 +351,9 @@ export class PluginHost {
             zipFile: this.zipFile,
             zipHash: this.zipHash,
         };
-        if (!needsClusterForkWorker(this.packageJson.scrypted)) {
+
+        // if a plugin requests a cluster worker, and it can be fulfilled by the server, do it.
+        if (!utilizesClusterForkWorker(this.packageJson.scrypted)) {
             this.peer = new RpcPeer('host', this.pluginId, (message, reject, serializationContext) => {
                 if (connected) {
                     this.worker.send(message, reject, serializationContext);

package/src/plugin/plugin-remote-worker.ts

@@ -4,10 +4,12 @@ import fs from 'fs';
 import path from 'path';
 import { install as installSourceMapSupport } from 'source-map-support';
 import worker_threads from 'worker_threads';
-import { needsClusterForkWorker } from '../cluster/cluster-labels';
+import { utilizesClusterForkWorker } from '../cluster/cluster-labels';
 import { setupCluster } from '../cluster/cluster-setup';
 import { RpcMessage, RpcPeer } from '../rpc';
 import { evalLocal } from '../rpc-peer-eval';
+import { ClusterManagerImpl } from '../scrypted-cluster-main';
+import type { PluginComponent } from '../services/plugin';
 import type { DeviceManagerImpl } from './device';
 import { MediaManagerImpl } from './media';
 import { PluginAPI, PluginAPIProxy, PluginRemote, PluginRemoteLoadZipOptions, PluginZipAPI } from './plugin-api';
@@ -22,9 +24,6 @@ import { NodeThreadWorker } from './runtime/node-thread-worker';
 import { prepareZip } from './runtime/node-worker-common';
 import { getBuiltinRuntimeHosts } from './runtime/runtime-host';
 import { RuntimeWorker, RuntimeWorkerOptions } from './runtime/runtime-worker';
-import type { ClusterForkService } from '../services/cluster-fork';
-import type { PluginComponent } from '../services/plugin';
-import { ClusterManagerImpl } from '../scrypted-cluster-main';
 
 const serverVersion = require('../../package.json').version;
 
@@ -226,7 +225,7 @@ export function startPluginRemote(mainFilename: string, pluginId: string, peerSe
                 };
 
                 // if running in a cluster, fork to a matching cluster worker only if necessary.
-                if (needsClusterForkWorker(options)) {
+                if (utilizesClusterForkWorker(options)) {
                     ({ runtimeWorker, forkPeer, clusterWorkerId } = createClusterForkWorker(
                         runtimeWorkerOptions,
                         options,

package/src/scrypted-cluster-main.ts

@@ -74,7 +74,9 @@ export interface ClusterWorkerProperties {
 
 export interface RunningClusterWorker extends ClusterWorkerProperties {
     id: string;
+    name: string;
     peer: RpcPeer;
+    fork: Promise<ClusterForkParam>;
     forks: Set<ClusterForkOptions>;
     address: string;
 }
@@ -113,6 +115,88 @@ export interface ClusterForkResultInterface {
 
 export type ClusterForkParam = (runtime: string, options: RuntimeWorkerOptions, peerLiveness: PeerLiveness, getZip: () => Promise<Buffer>) => Promise<ClusterForkResultInterface>;
 
+function createClusterForkParam(mainFilename: string, clusterId: string, clusterSecret: string) {
+    const clusterForkParam: ClusterForkParam = async (runtime, runtimeWorkerOptions, peerLiveness, getZip) => {
+        let runtimeWorker: RuntimeWorker;
+
+        const builtins = getBuiltinRuntimeHosts();
+        const rt = builtins.get(runtime);
+        if (!rt)
+            throw new Error('unknown runtime ' + runtime);
+
+        const pluginId: string = runtimeWorkerOptions.packageJson.name;
+        const { zipFile, unzippedPath } = await prepareZip(getPluginVolume(pluginId), runtimeWorkerOptions.zipHash, getZip);
+
+        const volume = getScryptedVolume();
+        const pluginVolume = getPluginVolume(pluginId);
+
+        runtimeWorkerOptions.zipFile = zipFile;
+        runtimeWorkerOptions.unzippedPath = unzippedPath;
+
+        runtimeWorkerOptions.env = {
+            ...runtimeWorkerOptions.env,
+            SCRYPTED_VOLUME: volume,
+            SCRYPTED_PLUGIN_VOLUME: pluginVolume,
+        };
+
+        runtimeWorker = rt(mainFilename, runtimeWorkerOptions, undefined);
+        runtimeWorker.stdout.on('data', data => console.log(data.toString()));
+        runtimeWorker.stderr.on('data', data => console.error(data.toString()));
+
+        const threadPeer = new RpcPeer('main', 'thread', (message, reject, serializationContext) => runtimeWorker.send(message, reject, serializationContext));
+        runtimeWorker.setupRpcPeer(threadPeer);
+        runtimeWorker.on('exit', () => {
+            threadPeer.kill('worker exited');
+        });
+        runtimeWorker.on('error', e => {
+            threadPeer.kill('worker error ' + e);
+        });
+        threadPeer.killedSafe.finally(() => {
+            runtimeWorker.kill();
+        });
+        peerLiveness.waitKilled().catch(() => { }).finally(() => {
+            threadPeer.kill('peer killed');
+        });
+        let getRemote: any;
+        let ping: any;
+        try {
+            const initializeCluster: InitializeCluster = await threadPeer.getParam('initializeCluster');
+            await initializeCluster({ clusterId, clusterSecret });
+            getRemote = await threadPeer.getParam('getRemote');
+            ping = await threadPeer.getParam('ping');
+        }
+        catch (e) {
+            threadPeer.kill('cluster fork failed');
+            throw e;
+        }
+
+        const readStream = async function* (stream: Readable) {
+            for await (const buffer of stream) {
+                yield buffer;
+            }
+        }
+
+        const timeout = setTimeout(() => {
+            threadPeer.kill('cluster fork timeout');
+        }, 10000);
+        const clusterGetRemote = (...args: any[]) => {
+            clearTimeout(timeout);
+            return {
+                [RpcPeer.PROPERTY_JSON_COPY_SERIALIZE_CHILDREN]: true,
+                stdout: readStream(runtimeWorker.stdout),
+                stderr: readStream(runtimeWorker.stderr),
+                getRemote,
+                ping,
+            };
+        };
+
+        const result = new ClusterForkResult(threadPeer, threadPeer.killed, clusterGetRemote);
+        return result;
+    };
+
+    return clusterForkParam;
+}
+
 export function startClusterClient(mainFilename: string) {
     console.log('Cluster client starting.');
     const originalClusterAddress = process.env.SCRYPTED_CLUSTER_ADDRESS;
@@ -174,7 +258,7 @@ export function startClusterClient(mainFilename: string) {
                 const auth: ClusterObject = {
                     address: socket.localAddress,
                     port: socket.localPort,
-                    id: process.env.SCRYPTED_CLUSTER_CLIENT_NAME || os.hostname(),
+                    id: process.env.SCRYPTED_CLUSTER_WORKER_NAME || os.hostname(),
                     proxyId: undefined,
                     sourceKey: undefined,
                     sha256: undefined,
@@ -190,85 +274,7 @@ export function startClusterClient(mainFilename: string) {
                 const clusterPeerSetup = setupCluster(peer);
                 await clusterPeerSetup.initializeCluster({ clusterId, clusterSecret });
 
-                const clusterForkParam: ClusterForkParam = async (runtime, runtimeWorkerOptions, peerLiveness, getZip) => {
-                    let runtimeWorker: RuntimeWorker;
-
-                    const builtins = getBuiltinRuntimeHosts();
-                    const rt = builtins.get(runtime);
-                    if (!rt)
-                        throw new Error('unknown runtime ' + runtime);
-
-                    const pluginId: string = runtimeWorkerOptions.packageJson.name;
-                    const { zipFile, unzippedPath } = await prepareZip(getPluginVolume(pluginId), runtimeWorkerOptions.zipHash, getZip);
-
-                    const volume = getScryptedVolume();
-                    const pluginVolume = getPluginVolume(pluginId);
-
-                    runtimeWorkerOptions.zipFile = zipFile;
-                    runtimeWorkerOptions.unzippedPath = unzippedPath;
-
-                    runtimeWorkerOptions.env = {
-                        ...runtimeWorkerOptions.env,
-                        SCRYPTED_VOLUME: volume,
-                        SCRYPTED_PLUGIN_VOLUME: pluginVolume,
-                    };
-
-                    runtimeWorker = rt(mainFilename, runtimeWorkerOptions, undefined);
-                    runtimeWorker.stdout.on('data', data => console.log(data.toString()));
-                    runtimeWorker.stderr.on('data', data => console.error(data.toString()));
-
-                    const threadPeer = new RpcPeer('main', 'thread', (message, reject, serializationContext) => runtimeWorker.send(message, reject, serializationContext));
-                    runtimeWorker.setupRpcPeer(threadPeer);
-                    runtimeWorker.on('exit', () => {
-                        threadPeer.kill('worker exited');
-                    });
-                    runtimeWorker.on('error', e => {
-                        threadPeer.kill('worker error ' + e);
-                    });
-                    threadPeer.killedSafe.finally(() => {
-                        runtimeWorker.kill();
-                    });
-                    peerLiveness.waitKilled().catch(() => { }).finally(() => {
-                        threadPeer.kill('peer killed');
-                    });
-                    let getRemote: any;
-                    let ping: any;
-                    try {
-                        const initializeCluster: InitializeCluster = await threadPeer.getParam('initializeCluster');
-                        await initializeCluster({ clusterId, clusterSecret });
-                        getRemote = await threadPeer.getParam('getRemote');
-                        ping = await threadPeer.getParam('ping');
-                    }
-                    catch (e) {
-                        threadPeer.kill('cluster fork failed');
-                        throw e;
-                    }
-
-                    const readStream = async function* (stream: Readable) {
-                        for await (const buffer of stream) {
-                            yield buffer;
-                        }
-                    }
-
-                    const timeout = setTimeout(() => {
-                        threadPeer.kill('cluster fork timeout');
-                    }, 10000);
-                    const clusterGetRemote = (...args: any[]) => {
-                        clearTimeout(timeout);
-                        return {
-                            [RpcPeer.PROPERTY_JSON_COPY_SERIALIZE_CHILDREN]: true,
-                            stdout: readStream(runtimeWorker.stdout),
-                            stderr: readStream(runtimeWorker.stderr),
-                            getRemote,
-                            ping,
-                        };
-                    };
-
-                    const result = new ClusterForkResult(threadPeer, threadPeer.killed, clusterGetRemote);
-                    return result;
-                };
-
-                peer.params['fork'] = clusterForkParam;
+                peer.params['fork'] = createClusterForkParam(mainFilename, clusterId, clusterSecret);
 
                 await peer.killed;
             }
@@ -284,7 +290,20 @@ export function startClusterClient(mainFilename: string) {
     })();
 }
 
-export function createClusterServer(runtime: ScryptedRuntime, certificate: ReturnType<typeof createSelfSignedCertificate>) {
+export function createClusterServer(mainFilename: string, scryptedRuntime: ScryptedRuntime, certificate: ReturnType<typeof createSelfSignedCertificate>) {
+    const serverClusterWorkerId = crypto.randomUUID();
+    process.env.SCRYPTED_CLUSTER_WORKER_ID = serverClusterWorkerId;
+    const serverWorker: RunningClusterWorker = {
+        labels: getClusterLabels(),
+        id: serverClusterWorkerId,
+        peer: undefined,
+        fork: Promise.resolve(createClusterForkParam(mainFilename, scryptedRuntime.clusterId, scryptedRuntime.clusterSecret)),
+        name: process.env.SCRYPTED_CLUSTER_WORKER_NAME || os.hostname(),
+        address: process.env.SCRYPTED_CLUSTER_ADDRESS,
+        forks: new Set(),
+    };
+    scryptedRuntime.clusterWorkers.set(serverClusterWorkerId, serverWorker);
+
     const server = tls.createServer({
         key: certificate.serviceKey,
         cert: certificate.certificate,
@@ -299,7 +318,7 @@ export function createClusterServer(runtime: ScryptedRuntime, certificate: Retur
         const connectForkWorker: ConnectForkWorker = async (auth: ClusterObject, properties: ClusterWorkerProperties) => {
             const id = crypto.randomUUID();
             try {
-                const sha256 = computeClusterObjectHash(auth, runtime.clusterSecret);
+                const sha256 = computeClusterObjectHash(auth, scryptedRuntime.clusterSecret);
                 if (sha256 !== auth.sha256)
                     throw new Error('cluster object hash mismatch');
 
@@ -313,18 +332,19 @@ export function createClusterServer(runtime: ScryptedRuntime, certificate: Retur
                 }
                 const worker: RunningClusterWorker = {
                     ...properties,
-                    // generate a random uuid.
                     id,
                     peer,
+                    fork: undefined,
+                    name: auth.id,
                     address: socket.remoteAddress,
                     forks: new Set(),
                 };
-                runtime.clusterWorkers.set(id, worker);
+                scryptedRuntime.clusterWorkers.set(id, worker);
                 peer.killedSafe.finally(() => {
-                    runtime.clusterWorkers.delete(id);
+                    scryptedRuntime.clusterWorkers.delete(id);
                 });
                 socket.on('close', () => {
-                    runtime.clusterWorkers.delete(id);
+                    scryptedRuntime.clusterWorkers.delete(id);
                 });
                 console.log('Cluster client authenticated.', socket.remoteAddress, socket.remotePort, properties);
             }
@@ -334,7 +354,7 @@ export function createClusterServer(runtime: ScryptedRuntime, certificate: Retur
             }
 
             return {
-                clusterId: runtime.clusterId,
+                clusterId: scryptedRuntime.clusterId,
                 clusterWorkerId: id,
             }
         }
@@ -345,7 +365,8 @@ export function createClusterServer(runtime: ScryptedRuntime, certificate: Retur
 }
 
 export class ClusterManagerImpl implements ClusterManager {
-    private  clusterServicePromise: Promise<ClusterForkService>;
+    private clusterServicePromise: Promise<ClusterForkService>;
+    private clusterMode = getScryptedClusterMode()[0];
 
     constructor(private api: PluginAPI) {
     }
@@ -355,7 +376,7 @@ export class ClusterManagerImpl implements ClusterManager {
     }
 
     getClusterMode(): 'server' | 'client' | undefined {
-        return getScryptedClusterMode()[0];
+        return this.clusterMode;
     }
 
     async getClusterWorkers(): Promise<Record<string, ClusterWorker>> {

package/src/scrypted-server-main.ts

@@ -23,7 +23,7 @@ import { getNpmPackageInfo } from './services/plugin';
 import { setScryptedUserPassword, UsersService } from './services/users';
 import { sleep } from './sleep';
 import { ONE_DAY_MILLISECONDS, UserToken } from './usertoken';
-import { createClusterServer } from './scrypted-cluster-main';
+import { createClusterServer, startClusterClient } from './scrypted-cluster-main';
 import { getScryptedClusterMode } from './cluster/cluster-setup';
 
 export type Runtime = ScryptedRuntime;
@@ -351,6 +351,14 @@ async function start(mainFilename: string, options?: {
 
     const scrypted = new ScryptedRuntime(mainFilename, db, insecure, secure, app);
     await options?.onRuntimeCreated?.(scrypted);
+
+    const clusterMode = getScryptedClusterMode();
+    if (clusterMode?.[0] === 'server') {
+        console.log('Cluster server starting.');
+        const clusterServer = createClusterServer(mainFilename, scrypted, keyPair);
+        await listenServerPort('SCRYPTED_CLUSTER_SERVER', clusterMode[2], clusterServer);
+    }
+
     await scrypted.start();
 
 
@@ -736,13 +744,6 @@ async function start(mainFilename: string, options?: {
     await listenServerPort('SCRYPTED_SECURE_PORT', SCRYPTED_SECURE_PORT, secure);
     await listenServerPort('SCRYPTED_INSECURE_PORT', SCRYPTED_INSECURE_PORT, insecure);
 
-    const clusterMode = getScryptedClusterMode();
-    if (clusterMode?.[0] === 'server') {
-        console.log('Cluster server starting.');
-        const clusterServer = createClusterServer(scrypted, keyPair);
-        await listenServerPort('SCRYPTED_CLUSTER_SERVER', clusterMode[2], clusterServer);
-    }
-
     console.log('#######################################################');
     console.log(`Scrypted Volume           : ${volumeDir}`);
     console.log(`Scrypted Server (Local)   : https://localhost:${SCRYPTED_SECURE_PORT}/`);

package/src/services/cluster-fork.ts

@@ -14,7 +14,7 @@ class WrappedForkResult implements ClusterForkResultInterface {
     }
 
     async kill() {
-        const fr = await this.forkResult.catch(() => {});
+        const fr = await this.forkResult.catch(() => { });
         if (!fr)
             return;
         await fr.kill();
@@ -35,7 +35,7 @@ export class ClusterForkService {
     constructor(public runtime: ScryptedRuntime) { }
 
     async fork(runtimeWorkerOptions: RuntimeWorkerOptions, options: ClusterForkOptions, peerLiveness: PeerLiveness, getZip: () => Promise<Buffer>) {
-        const matchingWorkers = [...this.runtime.clusterWorkers.entries()].map(([id, worker]) => ({
+        let matchingWorkers = [...this.runtime.clusterWorkers.entries()].map(([id, worker]) => ({
             worker,
             matches: matchesClusterLabels(options, worker.labels),
         }))
@@ -44,7 +44,6 @@ export class ClusterForkService {
                 // and worker id must match if provided
                 return matches && (!options.clusterWorkerId || worker.id === options.clusterWorkerId);
             });
-        matchingWorkers.sort((a, b) => b.worker.labels.length - a.worker.labels.length);
 
         let worker: RunningClusterWorker;
 
@@ -53,16 +52,31 @@ export class ClusterForkService {
         if (options.id)
             worker = matchingWorkers.find(({ worker }) => [...worker.forks].find(f => f.id === options.id))?.worker;
 
-        // TODO: round robin?
-        worker ||= matchingWorkers[0]?.worker;
-
         if (!worker) {
-            if (options.clusterWorkerId)
-                throw new Error(`no worker found for cluster id ${options.clusterWorkerId}`);
-            throw new Error(`no worker found for cluster labels ${JSON.stringify(options.labels)}`);
+            // sort by number of matches, to find the best match.
+            matchingWorkers.sort((a, b) => b.matches - a.matches);
+
+            const bestMatch = matchingWorkers[0];
+
+            if (!bestMatch) {
+                if (options.clusterWorkerId)
+                    throw new Error(`no worker found for cluster id ${options.clusterWorkerId}`);
+                throw new Error(`no worker found for cluster labels ${JSON.stringify(options.labels)}`);
+            }
+
+            // filter out workers that are not equivalent to the best match.
+            // this enforces the "prefer" label.
+            matchingWorkers = matchingWorkers.filter(({ matches }) => matches === bestMatch.matches)
+                // sort by number of forks, to distribute load.
+                .sort((a, b) => a.worker.forks.size - b.worker.forks.size);
+
+            worker = matchingWorkers[0]?.worker;
         }
 
-        const fork: ClusterForkParam = await worker.peer.getParam('fork');
+        console.log('forking to worker', worker.id, options);
+
+        worker.fork ||= worker.peer.getParam('fork');
+        const fork: ClusterForkParam = await worker.fork;
         const forkResultPromise = fork(options.runtime, runtimeWorkerOptions, peerLiveness, getZip);
 
         options.id ||= this.runtime.findPluginDevice(runtimeWorkerOptions.packageJson.name)?._id;
@@ -83,7 +97,7 @@ export class ClusterForkService {
         const ret: any = {};
         for (const worker of this.runtime.clusterWorkers.values()) {
             ret[worker.id] = {
-                name: worker.peer.peerName,
+                name: worker.name,
                 labels: worker.labels,
                 forks: [...worker.forks],
             };