@scrypted/server 0.123.46 → 0.123.48

package/src/cluster/connect-rpc-object.ts

@@ -1,9 +1,27 @@
 export interface ClusterObject {
+    /**
+     * Id of the cluster.
+     */
     id: string;
+    /**
+     * Address of the process that created this object.
+     */
     address: string;
+    /**
+     * Port of the process that created this object.
+     */
     port: number;
+    /**
+     * Id of the object within the source peer.
+     */
     proxyId: string;
+    /**
+     * Id of the source peer.
+     */
     sourceKey: string;
+    /**
+     * Hash of the object.
+     */
     sha256: string;
 }
 

package/src/cluster/cpu-timer.ts

@@ -0,0 +1,48 @@
+import { CpuInfo, cpus } from 'os';
+
+function getIdleTotal(cpu: CpuInfo) {
+    const t = cpu.times;
+    const total = t.user + t.nice + t.sys + t.idle + t.irq;
+    const idle = t.idle;
+    return {
+        idle,
+        total,
+    }
+}
+
+export class CpuTimer {
+    previousSample: ReturnType<typeof cpus>;
+    maxSpeed = 0;
+
+    sample(): number {
+        const sample = cpus();
+        const previousSample = this.previousSample;
+        this.previousSample = sample;
+
+        // can cpu count change at runtime, who knows
+        if (!previousSample || previousSample.length !== sample.length)
+            return 0;
+
+        // cpu may be throttled in low power mode, so observe total speed to scale
+        let totalSpeed = 0;
+
+        const times = sample.map((v, i) => {
+            totalSpeed += v.speed;
+            const c = getIdleTotal(v);
+            const p = getIdleTotal(previousSample[i]);
+            const total = c.total - p.total;
+            const idle = c.idle - p.idle;
+            return 1 - idle / total;
+        });
+
+        this.maxSpeed = Math.max(this.maxSpeed, totalSpeed);
+
+        // will return a value between 0 and 1, where 1 is full cpu speed
+        // the cpu usage is scaled by the clock speed
+        // so if the cpu is running at 1ghz out of 3ghz, the cpu usage is scaled by 1/3
+        const clockScale = totalSpeed / this.maxSpeed;
+
+        const total = times.reduce((p, c) => p + c, 0);
+        return total / sample.length * clockScale;
+    }
+}

package/src/plugin/plugin-api.ts

@@ -164,6 +164,7 @@ export interface PluginRemoteLoadZipOptions {
     main?: string;
 
     clusterId: string;
+    clusterWorkerId: string;
     clusterSecret: string;
 }
 

package/src/plugin/plugin-host.ts

@@ -248,6 +248,7 @@ export class PluginHost {
             const loadZipOptions: PluginRemoteLoadZipOptions = {
                 clusterId: this.scrypted.clusterId,
                 clusterSecret: this.scrypted.clusterSecret,
+                clusterWorkerId: await this.clusterWorkerId,
                 // debug flag can be used to affect path resolution for sourcemaps etc.
                 debug: !!pluginDebug,
                 zipHash: this.zipHash,
@@ -390,6 +391,7 @@ export class PluginHost {
                     await clusterSetup.initializeCluster({
                         clusterId: this.scrypted.clusterId,
                         clusterSecret: this.scrypted.clusterSecret,
+                        clusterWorkerId: this.scrypted.serverClusterWorkerId,
                     });
                     return this.scrypted.clusterFork;
                 })(),

package/src/plugin/plugin-remote-worker.ts

@@ -115,7 +115,7 @@ export function startPluginRemote(mainFilename: string, pluginId: string, peerSe
             await initializeCluster(zipOptions);
 
             scrypted.connectRPCObject = connectRPCObject;
-            scrypted.clusterManager = new ClusterManagerImpl(api);
+            scrypted.clusterManager = new ClusterManagerImpl(api, zipOptions.clusterWorkerId);
 
             if (worker_threads.isMainThread) {
                 const fsDir = path.join(unzippedPath, 'fs')
@@ -332,6 +332,7 @@ export function startPluginRemote(mainFilename: string, pluginId: string, peerSe
                     }
 
                     const forkOptions = Object.assign({}, zipOptions);
+                    forkOptions.clusterWorkerId = await clusterWorkerId || forkOptions.clusterWorkerId;
                     forkOptions.fork = true;
                     forkOptions.main = options?.filename;
                     const forkZipAPI = new PluginZipAPI(() => zipAPI.getZip());

package/src/runtime.ts

@@ -65,6 +65,7 @@ export class ScryptedRuntime extends PluginHttp<HttpPluginData> {
     clusterId = crypto.randomBytes(3).toString('hex');
     clusterSecret = process.env.SCRYPTED_CLUSTER_SECRET || crypto.randomBytes(16).toString('hex');
     clusterWorkers = new Map<string, RunningClusterWorker>();
+    serverClusterWorkerId: string;
     plugins: { [id: string]: PluginHost } = {};
     pluginDevices: { [id: string]: PluginDevice } = {};
     devices: { [id: string]: DeviceProxyPair } = {};

package/src/scrypted-cluster-main.ts

@@ -1,4 +1,4 @@
-import type { ClusterManager, ClusterWorker, ForkOptions } from '@scrypted/types';
+import type { ClusterManager, ForkOptions } from '@scrypted/types';
 import crypto from 'crypto';
 import { once } from 'events';
 import net from 'net';
@@ -8,9 +8,10 @@ import type { Readable } from 'stream';
 import tls from 'tls';
 import type { createSelfSignedCertificate } from './cert';
 import { computeClusterObjectHash } from './cluster/cluster-hash';
-import { getClusterLabels } from './cluster/cluster-labels';
+import { getClusterLabels, getClusterWorkerWeight } from './cluster/cluster-labels';
 import { getScryptedClusterMode, InitializeCluster, setupCluster } from './cluster/cluster-setup';
 import type { ClusterObject } from './cluster/connect-rpc-object';
+import { CpuTimer } from './cluster/cpu-timer';
 import type { PluginAPI } from './plugin/plugin-api';
 import { getPluginVolume, getScryptedVolume } from './plugin/plugin-volume';
 import { prepareZip } from './plugin/runtime/node-worker-common';
@@ -20,10 +21,10 @@ import { RpcPeer } from './rpc';
 import { createRpcDuplexSerializer } from './rpc-serializer';
 import type { ScryptedRuntime } from './runtime';
 import type { ClusterForkService } from './services/cluster-fork';
-import { sleep } from './sleep';
-import type { ServiceControl } from './services/service-control';
 import { EnvControl } from './services/env';
 import { Info } from './services/info';
+import type { ServiceControl } from './services/service-control';
+import { sleep } from './sleep';
 
 installSourceMapSupport({
     environment: 'node',
@@ -73,6 +74,7 @@ type ConnectForkWorker = (auth: ClusterObject, properties: ClusterWorkerProperti
 
 export interface ClusterWorkerProperties {
     labels: string[];
+    weight: number;
 }
 
 export interface RunningClusterWorker extends ClusterWorkerProperties {
@@ -82,6 +84,8 @@ export interface RunningClusterWorker extends ClusterWorkerProperties {
     fork: Promise<ClusterForkParam>;
     forks: Set<ClusterForkOptions>;
     address: string;
+    weight: number;
+    cpuUsage: number;
 }
 
 export class PeerLiveness {
@@ -118,7 +122,7 @@ export interface ClusterForkResultInterface {
 
 export type ClusterForkParam = (runtime: string, options: RuntimeWorkerOptions, peerLiveness: PeerLiveness, getZip: () => Promise<Buffer>) => Promise<ClusterForkResultInterface>;
 
-function createClusterForkParam(mainFilename: string, clusterId: string, clusterSecret: string) {
+function createClusterForkParam(mainFilename: string, clusterId: string, clusterSecret: string, clusterWorkerId: string) {
     const clusterForkParam: ClusterForkParam = async (runtime, runtimeWorkerOptions, peerLiveness, getZip) => {
         let runtimeWorker: RuntimeWorker;
 
@@ -164,7 +168,7 @@ function createClusterForkParam(mainFilename: string, clusterId: string, cluster
         let ping: any;
         try {
             const initializeCluster: InitializeCluster = await threadPeer.getParam('initializeCluster');
-            await initializeCluster({ clusterId, clusterSecret });
+            await initializeCluster({ clusterId, clusterSecret, clusterWorkerId });
             getRemote = await threadPeer.getParam('getRemote');
             ping = await threadPeer.getParam('ping');
         }
@@ -204,9 +208,11 @@ export function startClusterClient(mainFilename: string, serviceControl?: Servic
     console.log('Cluster client starting.');
 
     const envControl = new EnvControl();
+    const cpuTimer = new CpuTimer();
 
     const originalClusterAddress = process.env.SCRYPTED_CLUSTER_ADDRESS;
     const labels = getClusterLabels();
+    const weight = getClusterWorkerWeight();
 
     const clusterSecret = process.env.SCRYPTED_CLUSTER_SECRET;
     const clusterMode = getScryptedClusterMode();
@@ -256,6 +262,7 @@ export function startClusterClient(mainFilename: string, serviceControl?: Servic
             peer.params['service-control'] = serviceControl;
             peer.params['env-control'] = envControl;
             peer.params['info'] = new Info();
+            peer.params['cpu'] = async () => cpuTimer.sample();
 
             const { localAddress, localPort } = socket;
             console.log('Cluster server connected.', localAddress, localPort);
@@ -277,14 +284,14 @@ export function startClusterClient(mainFilename: string, serviceControl?: Servic
 
                 const properties: ClusterWorkerProperties = {
                     labels,
+                    weight,
                 };
 
                 const { clusterId, clusterWorkerId } = await connectForkWorker(auth, properties);
-                process.env.SCRYPTED_CLUSTER_WORKER_ID = clusterWorkerId;
                 const clusterPeerSetup = setupCluster(peer);
-                await clusterPeerSetup.initializeCluster({ clusterId, clusterSecret });
+                await clusterPeerSetup.initializeCluster({ clusterId, clusterSecret, clusterWorkerId });
 
-                peer.params['fork'] = createClusterForkParam(mainFilename, clusterId, clusterSecret);
+                peer.params['fork'] = createClusterForkParam(mainFilename, clusterId, clusterSecret, clusterWorkerId);
 
                 await peer.killed;
             }
@@ -301,18 +308,26 @@ export function startClusterClient(mainFilename: string, serviceControl?: Servic
 }
 
 export function createClusterServer(mainFilename: string, scryptedRuntime: ScryptedRuntime, certificate: ReturnType<typeof createSelfSignedCertificate>) {
-    const serverClusterWorkerId = crypto.randomUUID();
-    process.env.SCRYPTED_CLUSTER_WORKER_ID = serverClusterWorkerId;
+    scryptedRuntime.serverClusterWorkerId = crypto.randomUUID();
     const serverWorker: RunningClusterWorker = {
         labels: getClusterLabels(),
-        id: serverClusterWorkerId,
+        id: scryptedRuntime.serverClusterWorkerId,
         peer: undefined,
-        fork: Promise.resolve(createClusterForkParam(mainFilename, scryptedRuntime.clusterId, scryptedRuntime.clusterSecret)),
+        fork: Promise.resolve(createClusterForkParam(mainFilename, scryptedRuntime.clusterId, scryptedRuntime.clusterSecret, scryptedRuntime.serverClusterWorkerId)),
         name: process.env.SCRYPTED_CLUSTER_WORKER_NAME || os.hostname(),
         address: process.env.SCRYPTED_CLUSTER_ADDRESS,
+        weight: getClusterWorkerWeight(),
         forks: new Set(),
+        cpuUsage: 0,
     };
-    scryptedRuntime.clusterWorkers.set(serverClusterWorkerId, serverWorker);
+    scryptedRuntime.clusterWorkers.set(scryptedRuntime.serverClusterWorkerId, serverWorker);
+
+    {
+        const cpuTimer = new CpuTimer();
+        setInterval(() => {
+            serverWorker.cpuUsage = cpuTimer.sample();
+        }, 1000);
+    }
 
     const server = tls.createServer({
         key: certificate.serviceKey,
@@ -348,6 +363,7 @@ export function createClusterServer(mainFilename: string, scryptedRuntime: Scryp
                     name: auth.id,
                     address: socket.remoteAddress,
                     forks: new Set(),
+                    cpuUsage: 0,
                 };
                 scryptedRuntime.clusterWorkers.set(id, worker);
                 peer.killedSafe.finally(() => {
@@ -357,6 +373,16 @@ export function createClusterServer(mainFilename: string, scryptedRuntime: Scryp
                     scryptedRuntime.clusterWorkers.delete(id);
                 });
                 console.log('Cluster client authenticated.', socket.remoteAddress, socket.remotePort, properties);
+
+                let cpu: Promise<() => Promise<number>>;
+                const cpuTimer = setInterval(async () => {
+                    cpu ||= peer.getParam('cpu');
+                    const usage = await (await cpu)();
+                    worker.cpuUsage = usage;
+                }, 1000);
+                peer.killedSafe.finally(() => {
+                    clearInterval(cpuTimer);
+                });
             }
             catch (e) {
                 peer.kill(e);
@@ -378,18 +404,18 @@ export class ClusterManagerImpl implements ClusterManager {
     private clusterServicePromise: Promise<ClusterForkService>;
     private clusterMode = getScryptedClusterMode()?.[0];
 
-    constructor(private api: PluginAPI) {
+    constructor(private api: PluginAPI, private clusterWorkerId: string) {
     }
 
     getClusterWorkerId(): string {
-        return process.env.SCRYPTED_CLUSTER_WORKER_ID;
+        return this.clusterWorkerId;
     }
 
     getClusterMode(): 'server' | 'client' | undefined {
         return this.clusterMode;
     }
 
-    async getClusterWorkers(): Promise<Record<string, ClusterWorker>> {
+    async getClusterWorkers() {
         const clusterFork = await this.getClusterService();
         return clusterFork.getClusterWorkers();
     }

package/src/services/cluster-fork.ts

@@ -1,3 +1,4 @@
+import { ClusterFork, ClusterWorker } from "@scrypted/types";
 import { matchesClusterLabels } from "../cluster/cluster-labels";
 import type { RuntimeWorkerOptions } from "../plugin/runtime/runtime-worker";
 import { RpcPeer } from "../rpc";
@@ -68,7 +69,7 @@ export class ClusterForkService {
             // this enforces the "prefer" label.
             matchingWorkers = matchingWorkers.filter(({ matches }) => matches === bestMatch.matches)
                 // sort by number of forks, to distribute load.
-                .sort((a, b) => a.worker.forks.size - b.worker.forks.size);
+                .sort((a, b) => a.worker.forks.size * a.worker.weight - b.worker.forks.size * b.worker.weight);
 
             worker = matchingWorkers[0]?.worker;
         }
@@ -98,13 +99,15 @@ export class ClusterForkService {
         return ret;
     };
 
-    async getClusterWorkers() {
-        const ret: any = {};
+    async getClusterWorkers(): Promise<Record<string, ClusterWorker>> {
+        const ret: Record<string, ClusterWorker> = {};
         for (const worker of this.runtime.clusterWorkers.values()) {
             ret[worker.id] = {
+                id: worker.id,
                 name: worker.name,
                 labels: worker.labels,
-                forks: [...worker.forks],
+                forks: [...worker.forks] as ClusterFork[],
+                cpuUsage: worker.cpuUsage,
             };
         }
         return ret;