@scrypted/server 0.123.9 → 0.123.10
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cluster/cluster-labels.d.ts +3 -0
- package/dist/cluster/cluster-labels.js +39 -0
- package/dist/cluster/cluster-labels.js.map +1 -0
- package/dist/{scrypted-cluster-common.d.ts → cluster/cluster-setup.d.ts} +8 -6
- package/dist/{scrypted-cluster-common.js → cluster/cluster-setup.js} +52 -11
- package/dist/cluster/cluster-setup.js.map +1 -0
- package/dist/plugin/plugin-console.js +2 -2
- package/dist/plugin/plugin-console.js.map +1 -1
- package/dist/plugin/plugin-host.js +3 -3
- package/dist/plugin/plugin-host.js.map +1 -1
- package/dist/plugin/plugin-remote-worker.js +2 -2
- package/dist/plugin/plugin-remote-worker.js.map +1 -1
- package/dist/plugin/runtime/cluster-fork.worker.d.ts +1 -1
- package/dist/plugin/runtime/cluster-fork.worker.js +4 -3
- package/dist/plugin/runtime/cluster-fork.worker.js.map +1 -1
- package/dist/plugin/runtime/node-fork-worker.js +2 -2
- package/dist/plugin/runtime/node-fork-worker.js.map +1 -1
- package/dist/runtime.d.ts +1 -1
- package/dist/runtime.js +2 -2
- package/dist/runtime.js.map +1 -1
- package/dist/scrypted-cluster-main.d.ts +31 -0
- package/dist/scrypted-cluster-main.js +232 -2
- package/dist/scrypted-cluster-main.js.map +1 -1
- package/dist/scrypted-main-exports.js +2 -2
- package/dist/scrypted-main-exports.js.map +1 -1
- package/dist/scrypted-server-main.js +4 -3
- package/dist/scrypted-server-main.js.map +1 -1
- package/dist/services/cluster-fork.d.ts +2 -2
- package/dist/services/cluster-fork.js +2 -2
- package/dist/services/cluster-fork.js.map +1 -1
- package/package.json +1 -1
- package/python/plugin_remote.py +100 -141
- package/src/cluster/cluster-labels.ts +36 -0
- package/src/{scrypted-cluster-common.ts → cluster/cluster-setup.ts} +65 -13
- package/src/plugin/plugin-console.ts +1 -1
- package/src/plugin/plugin-host.ts +4 -4
- package/src/plugin/plugin-remote-worker.ts +2 -7
- package/src/plugin/runtime/cluster-fork.worker.ts +2 -2
- package/src/plugin/runtime/node-fork-worker.ts +1 -1
- package/src/runtime.ts +2 -2
- package/src/scrypted-cluster-main.ts +280 -1
- package/src/scrypted-main-exports.ts +1 -1
- package/src/scrypted-server-main.ts +2 -1
- package/src/services/cluster-fork.ts +2 -1
- package/dist/scrypted-cluster-common.js.map +0 -1
- package/dist/scrypted-cluster.d.ts +0 -38
- package/dist/scrypted-cluster.js +0 -291
- package/dist/scrypted-cluster.js.map +0 -1
- package/src/scrypted-cluster.ts +0 -345
@@ -1,8 +1,8 @@
|
|
1
1
|
import type { ScryptedRuntime } from "../runtime";
|
2
|
-
import { ClusterForkOptions, PeerLiveness } from "../scrypted-cluster";
|
2
|
+
import { ClusterForkOptions, PeerLiveness } from "../scrypted-cluster-main";
|
3
3
|
export declare class ClusterFork {
|
4
4
|
runtime: ScryptedRuntime;
|
5
5
|
constructor(runtime: ScryptedRuntime);
|
6
|
-
fork(peerLiveness: PeerLiveness, options: ClusterForkOptions, packageJson: any, zipHash: string, getZip: () => Promise<Buffer>): Promise<import("../scrypted-cluster").ClusterForkResult>;
|
6
|
+
fork(peerLiveness: PeerLiveness, options: ClusterForkOptions, packageJson: any, zipHash: string, getZip: () => Promise<Buffer>): Promise<import("../scrypted-cluster-main").ClusterForkResult>;
|
7
7
|
getClusterWorkers(): Promise<any>;
|
8
8
|
}
|
@@ -1,7 +1,7 @@
|
|
1
1
|
"use strict";
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
3
3
|
exports.ClusterFork = void 0;
|
4
|
-
const
|
4
|
+
const cluster_labels_1 = require("../cluster/cluster-labels");
|
5
5
|
class ClusterFork {
|
6
6
|
runtime;
|
7
7
|
constructor(runtime) {
|
@@ -10,7 +10,7 @@ class ClusterFork {
|
|
10
10
|
async fork(peerLiveness, options, packageJson, zipHash, getZip) {
|
11
11
|
const matchingWorkers = [...this.runtime.clusterWorkers].map(worker => ({
|
12
12
|
worker,
|
13
|
-
matches: (0,
|
13
|
+
matches: (0, cluster_labels_1.matchesClusterLabels)(options, worker.labels),
|
14
14
|
}))
|
15
15
|
.filter(({ matches }) => matches);
|
16
16
|
matchingWorkers.sort((a, b) => b.worker.labels.length - a.worker.labels.length);
|
@@ -1 +1 @@
|
|
1
|
-
{"version":3,"file":"cluster-fork.js","sourceRoot":"","sources":["../../src/services/cluster-fork.ts"],"names":[],"mappings":";;;AACA,
|
1
|
+
{"version":3,"file":"cluster-fork.js","sourceRoot":"","sources":["../../src/services/cluster-fork.ts"],"names":[],"mappings":";;;AACA,8DAAiE;AAGjE,MAAa,WAAW;IACD;IAAnB,YAAmB,OAAwB;QAAxB,YAAO,GAAP,OAAO,CAAiB;IAAI,CAAC;IAEhD,KAAK,CAAC,IAAI,CAAC,YAA0B,EAAE,OAA2B,EAAE,WAAgB,EAAE,OAAe,EAAE,MAA6B;QAChI,MAAM,eAAe,GAAG,CAAC,GAAG,IAAI,CAAC,OAAO,CAAC,cAAc,CAAC,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;YACpE,MAAM;YACN,OAAO,EAAE,IAAA,qCAAoB,EAAC,OAAO,EAAE,MAAM,CAAC,MAAM,CAAC;SACxD,CAAC,CAAC;aACF,MAAM,CAAC,CAAC,EAAE,OAAO,EAAE,EAAE,EAAE,CAAC,OAAO,CAAC,CAAC;QAClC,eAAe,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;QAChF,MAAM,MAAM,GAAG,eAAe,CAAC,CAAC,CAAC,EAAE,MAAM,CAAC;QAE1C,IAAI,CAAC,MAAM;YACP,MAAM,IAAI,KAAK,CAAC,sCAAsC,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;QAE5F,MAAM,IAAI,GAAqB,MAAM,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;QAClE,OAAO,IAAI,CAAC,YAAY,EAAE,OAAO,CAAC,OAAO,EAAE,WAAW,EAAE,OAAO,EAAE,MAAM,CAAC,CAAC;IAC7E,CAAC;IAED,KAAK,CAAC,iBAAiB;QACnB,MAAM,GAAG,GAAQ,EAAE,CAAC;QACpB,KAAK,MAAM,MAAM,IAAI,IAAI,CAAC,OAAO,CAAC,cAAc,EAAE,CAAC;YAC/C,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC,GAAG;gBACxB,MAAM,EAAE,MAAM,CAAC,MAAM;aACxB,CAAC;QACN,CAAC;QACD,OAAO,GAAG,CAAC;IACf,CAAC;CACJ;AA5BD,kCA4BC"}
|
package/package.json
CHANGED
package/python/plugin_remote.py
CHANGED
@@ -11,7 +11,6 @@ import os
|
|
11
11
|
import platform
|
12
12
|
import random
|
13
13
|
import sys
|
14
|
-
import threading
|
15
14
|
import time
|
16
15
|
import traceback
|
17
16
|
import zipfile
|
@@ -582,7 +581,7 @@ class ClusterSetup():
|
|
582
581
|
return
|
583
582
|
return sourcePeer.localProxyMap.get(id, None)
|
584
583
|
|
585
|
-
async def
|
584
|
+
async def connectClusterObject(self, o: ClusterObject):
|
586
585
|
sha256 = self.computeClusterObjectHash(o)
|
587
586
|
if sha256 != o["sha256"]:
|
588
587
|
raise Exception("secret incorrect")
|
@@ -647,7 +646,7 @@ class ClusterSetup():
|
|
647
646
|
future: asyncio.Future[rpc.RpcPeer] = asyncio.Future()
|
648
647
|
future.set_result(peer)
|
649
648
|
self.clusterPeers[clusterPeerKey] = future
|
650
|
-
peer.params["connectRPCObject"] = lambda o: self.
|
649
|
+
peer.params["connectRPCObject"] = lambda o: self.connectClusterObject(o)
|
651
650
|
try:
|
652
651
|
await peerReadLoop()
|
653
652
|
except:
|
@@ -675,6 +674,93 @@ class ClusterSetup():
|
|
675
674
|
)
|
676
675
|
return base64.b64encode(m.digest()).decode("utf-8")
|
677
676
|
|
677
|
+
def ensureClusterPeer(self, address: str, port: int):
|
678
|
+
if isClusterAddress(address):
|
679
|
+
address = "127.0.0.1"
|
680
|
+
clusterPeerKey = getClusterPeerKey(address, port)
|
681
|
+
clusterPeerPromise = self.clusterPeers.get(clusterPeerKey)
|
682
|
+
if clusterPeerPromise:
|
683
|
+
return clusterPeerPromise
|
684
|
+
|
685
|
+
async def connectClusterPeer():
|
686
|
+
try:
|
687
|
+
reader, writer = await asyncio.open_connection(address, port)
|
688
|
+
sourceAddress, sourcePort = writer.get_extra_info("sockname")
|
689
|
+
if (
|
690
|
+
sourceAddress != self.SCRYPTED_CLUSTER_ADDRESS
|
691
|
+
and sourceAddress != "127.0.0.1"
|
692
|
+
):
|
693
|
+
print("source address mismatch", sourceAddress)
|
694
|
+
rpcTransport = rpc_reader.RpcStreamTransport(reader, writer)
|
695
|
+
clusterPeer, peerReadLoop = await rpc_reader.prepare_peer_readloop(
|
696
|
+
self.loop, rpcTransport
|
697
|
+
)
|
698
|
+
# set all params from self.peer
|
699
|
+
for key, value in self.peer.params.items():
|
700
|
+
clusterPeer.params[key] = value
|
701
|
+
clusterPeer.onProxySerialization = (
|
702
|
+
lambda value: self.clusterSetup.onProxySerialization(
|
703
|
+
clusterPeer, value, clusterPeerKey
|
704
|
+
)
|
705
|
+
)
|
706
|
+
except:
|
707
|
+
self.clusterPeers.pop(clusterPeerKey)
|
708
|
+
raise
|
709
|
+
|
710
|
+
async def run_loop():
|
711
|
+
try:
|
712
|
+
await peerReadLoop()
|
713
|
+
except:
|
714
|
+
pass
|
715
|
+
finally:
|
716
|
+
self.clusterPeers.pop(clusterPeerKey)
|
717
|
+
|
718
|
+
asyncio.run_coroutine_threadsafe(run_loop(), self.loop)
|
719
|
+
return clusterPeer
|
720
|
+
|
721
|
+
clusterPeerPromise = self.loop.create_task(connectClusterPeer())
|
722
|
+
|
723
|
+
self.clusterPeers[clusterPeerKey] = clusterPeerPromise
|
724
|
+
return clusterPeerPromise
|
725
|
+
|
726
|
+
async def connectRPCObject(self, value):
|
727
|
+
__cluster = getattr(value, "__cluster")
|
728
|
+
if type(__cluster) is not dict:
|
729
|
+
return value
|
730
|
+
|
731
|
+
clusterObject: ClusterObject = __cluster
|
732
|
+
|
733
|
+
if clusterObject.get("id", None) != self.clusterId:
|
734
|
+
return value
|
735
|
+
|
736
|
+
address = clusterObject.get("address", None)
|
737
|
+
port = clusterObject["port"]
|
738
|
+
proxyId = clusterObject["proxyId"]
|
739
|
+
if port == self.clusterPort:
|
740
|
+
return await self.connectRPCObject(clusterObject)
|
741
|
+
|
742
|
+
clusterPeerPromise = self.ensureClusterPeer(address, port)
|
743
|
+
|
744
|
+
try:
|
745
|
+
clusterPeer = await clusterPeerPromise
|
746
|
+
weakref = clusterPeer.remoteWeakProxies.get(proxyId, None)
|
747
|
+
existing = weakref() if weakref else None
|
748
|
+
if existing:
|
749
|
+
return existing
|
750
|
+
|
751
|
+
peerConnectRPCObject = clusterPeer.tags.get("connectRPCObject")
|
752
|
+
if not peerConnectRPCObject:
|
753
|
+
peerConnectRPCObject = await clusterPeer.getParam(
|
754
|
+
"connectRPCObject"
|
755
|
+
)
|
756
|
+
clusterPeer.tags["connectRPCObject"] = peerConnectRPCObject
|
757
|
+
newValue = await peerConnectRPCObject(clusterObject)
|
758
|
+
if not newValue:
|
759
|
+
raise Exception("rpc object not found?")
|
760
|
+
return newValue
|
761
|
+
except Exception as e:
|
762
|
+
return value
|
763
|
+
|
678
764
|
class PluginRemote:
|
679
765
|
def __init__(
|
680
766
|
self, clusterSetup: ClusterSetup, api, pluginId: str, hostInfo, loop: AbstractEventLoop
|
@@ -757,94 +843,7 @@ class PluginRemote:
|
|
757
843
|
|
758
844
|
sdk = ScryptedStatic()
|
759
845
|
|
760
|
-
|
761
|
-
if isClusterAddress(address):
|
762
|
-
address = "127.0.0.1"
|
763
|
-
clusterPeerKey = getClusterPeerKey(address, port)
|
764
|
-
clusterPeerPromise = self.clusterSetup.clusterPeers.get(clusterPeerKey)
|
765
|
-
if clusterPeerPromise:
|
766
|
-
return clusterPeerPromise
|
767
|
-
|
768
|
-
async def connectClusterPeer():
|
769
|
-
try:
|
770
|
-
reader, writer = await asyncio.open_connection(address, port)
|
771
|
-
sourceAddress, sourcePort = writer.get_extra_info("sockname")
|
772
|
-
if (
|
773
|
-
sourceAddress != self.clusterSetup.SCRYPTED_CLUSTER_ADDRESS
|
774
|
-
and sourceAddress != "127.0.0.1"
|
775
|
-
):
|
776
|
-
print("source address mismatch", sourceAddress)
|
777
|
-
rpcTransport = rpc_reader.RpcStreamTransport(reader, writer)
|
778
|
-
clusterPeer, peerReadLoop = await rpc_reader.prepare_peer_readloop(
|
779
|
-
self.loop, rpcTransport
|
780
|
-
)
|
781
|
-
# set all params from self.peer
|
782
|
-
for key, value in self.peer.params.items():
|
783
|
-
clusterPeer.params[key] = value
|
784
|
-
clusterPeer.onProxySerialization = (
|
785
|
-
lambda value: self.clusterSetup.onProxySerialization(
|
786
|
-
clusterPeer, value, clusterPeerKey
|
787
|
-
)
|
788
|
-
)
|
789
|
-
except:
|
790
|
-
self.clusterSetup.clusterPeers.pop(clusterPeerKey)
|
791
|
-
raise
|
792
|
-
|
793
|
-
async def run_loop():
|
794
|
-
try:
|
795
|
-
await peerReadLoop()
|
796
|
-
except:
|
797
|
-
pass
|
798
|
-
finally:
|
799
|
-
self.clusterSetup.clusterPeers.pop(clusterPeerKey)
|
800
|
-
|
801
|
-
asyncio.run_coroutine_threadsafe(run_loop(), self.loop)
|
802
|
-
return clusterPeer
|
803
|
-
|
804
|
-
clusterPeerPromise = self.loop.create_task(connectClusterPeer())
|
805
|
-
|
806
|
-
self.clusterSetup.clusterPeers[clusterPeerKey] = clusterPeerPromise
|
807
|
-
return clusterPeerPromise
|
808
|
-
|
809
|
-
async def connectRPCObject(value):
|
810
|
-
__cluster = getattr(value, "__cluster")
|
811
|
-
if type(__cluster) is not dict:
|
812
|
-
return value
|
813
|
-
|
814
|
-
clusterObject: ClusterObject = __cluster
|
815
|
-
|
816
|
-
if clusterObject.get("id", None) != self.clusterSetup.clusterId:
|
817
|
-
return value
|
818
|
-
|
819
|
-
address = clusterObject.get("address", None)
|
820
|
-
port = clusterObject["port"]
|
821
|
-
proxyId = clusterObject["proxyId"]
|
822
|
-
if port == self.clusterSetup.clusterPort:
|
823
|
-
return await self.clusterSetup.connectRPCObject(clusterObject)
|
824
|
-
|
825
|
-
clusterPeerPromise = ensureClusterPeer(address, port)
|
826
|
-
|
827
|
-
try:
|
828
|
-
clusterPeer = await clusterPeerPromise
|
829
|
-
weakref = clusterPeer.remoteWeakProxies.get(proxyId, None)
|
830
|
-
existing = weakref() if weakref else None
|
831
|
-
if existing:
|
832
|
-
return existing
|
833
|
-
|
834
|
-
peerConnectRPCObject = clusterPeer.tags.get("connectRPCObject")
|
835
|
-
if not peerConnectRPCObject:
|
836
|
-
peerConnectRPCObject = await clusterPeer.getParam(
|
837
|
-
"connectRPCObject"
|
838
|
-
)
|
839
|
-
clusterPeer.tags["connectRPCObject"] = peerConnectRPCObject
|
840
|
-
newValue = await peerConnectRPCObject(clusterObject)
|
841
|
-
if not newValue:
|
842
|
-
raise Exception("rpc object not found?")
|
843
|
-
return newValue
|
844
|
-
except Exception as e:
|
845
|
-
return value
|
846
|
-
|
847
|
-
sdk.connectRPCObject = connectRPCObject
|
846
|
+
sdk.connectRPCObject = lambda v: self.clusterSetup.connectRPCObject(v)
|
848
847
|
|
849
848
|
forkMain = options and options.get("fork")
|
850
849
|
debug = options.get("debug", None)
|
@@ -1142,15 +1141,6 @@ class PluginRemote:
|
|
1142
1141
|
raise Exception(f"unknown service {name}")
|
1143
1142
|
|
1144
1143
|
async def start_stats_runner(self, update_stats):
|
1145
|
-
pong = None
|
1146
|
-
|
1147
|
-
async def ping(time: int):
|
1148
|
-
nonlocal pong
|
1149
|
-
pong = pong or await self.peer.getParam("pong")
|
1150
|
-
await pong(time)
|
1151
|
-
|
1152
|
-
self.peer.params["ping"] = ping
|
1153
|
-
|
1154
1144
|
def stats_runner():
|
1155
1145
|
ptime = round(time.process_time() * 1000000) + self.ptimeSum
|
1156
1146
|
try:
|
@@ -1193,6 +1183,14 @@ async def plugin_async_main(
|
|
1193
1183
|
clusterSetup = ClusterSetup(loop, peer)
|
1194
1184
|
peer.params["initializeCluster"] = lambda options: clusterSetup.initializeCluster(options)
|
1195
1185
|
|
1186
|
+
pong = None
|
1187
|
+
async def ping(time: int):
|
1188
|
+
nonlocal pong
|
1189
|
+
pong = pong or await peer.getParam("pong")
|
1190
|
+
await pong(time)
|
1191
|
+
|
1192
|
+
peer.params["ping"] = ping
|
1193
|
+
|
1196
1194
|
peer.params["getRemote"] = lambda api, pluginId, hostInfo: PluginRemote(
|
1197
1195
|
clusterSetup, api, pluginId, hostInfo, loop
|
1198
1196
|
)
|
@@ -1215,48 +1213,9 @@ def main(rpcTransport: rpc_reader.RpcTransport):
|
|
1215
1213
|
loop.run_until_complete(plugin_async_main(loop, rpcTransport))
|
1216
1214
|
loop.close()
|
1217
1215
|
|
1218
|
-
|
1219
|
-
def plugin_main(rpcTransport: rpc_reader.RpcTransport):
|
1220
|
-
if True:
|
1221
|
-
main(rpcTransport)
|
1222
|
-
return
|
1223
|
-
|
1224
|
-
# 03/05/2024
|
1225
|
-
# Not sure why this code below was necessary. I thought it was gstreamer needing to
|
1226
|
-
# be initialized on the main thread, but that no longer seems to be the case.
|
1227
|
-
|
1228
|
-
# gi import will fail on windows (and posisbly elsewhere)
|
1229
|
-
# if it does, try starting without it.
|
1230
|
-
try:
|
1231
|
-
import gi
|
1232
|
-
|
1233
|
-
gi.require_version("Gst", "1.0")
|
1234
|
-
from gi.repository import GLib, Gst
|
1235
|
-
|
1236
|
-
Gst.init(None)
|
1237
|
-
|
1238
|
-
# can't remember why starting the glib main loop is necessary.
|
1239
|
-
# maybe gstreamer on linux and other things needed it?
|
1240
|
-
# seems optional on other platforms.
|
1241
|
-
loop = GLib.MainLoop()
|
1242
|
-
|
1243
|
-
worker = threading.Thread(
|
1244
|
-
target=main, args=(rpcTransport,), name="asyncio-main"
|
1245
|
-
)
|
1246
|
-
worker.start()
|
1247
|
-
|
1248
|
-
loop.run()
|
1249
|
-
return
|
1250
|
-
except:
|
1251
|
-
pass
|
1252
|
-
|
1253
|
-
# reattempt without gi outside of the exception handler in case the plugin fails.
|
1254
|
-
main(rpcTransport)
|
1255
|
-
|
1256
|
-
|
1257
1216
|
def plugin_fork(conn: multiprocessing.connection.Connection):
|
1258
|
-
|
1217
|
+
main(rpc_reader.RpcConnectionTransport(conn))
|
1259
1218
|
|
1260
1219
|
|
1261
1220
|
if __name__ == "__main__":
|
1262
|
-
|
1221
|
+
main(rpc_reader.RpcFileTransport(3, 4))
|
@@ -0,0 +1,36 @@
|
|
1
|
+
import os from 'os';
|
2
|
+
import { ClusterForkOptions } from "../scrypted-cluster-main";
|
3
|
+
|
4
|
+
export function matchesClusterLabels(options: ClusterForkOptions, labels: string[]) {
|
5
|
+
let matched = 0;
|
6
|
+
for (const label of options?.labels?.require || []) {
|
7
|
+
if (!labels.includes(label))
|
8
|
+
return 0;
|
9
|
+
}
|
10
|
+
|
11
|
+
// if there is nothing in the any list, consider it matched
|
12
|
+
let foundAny = !options?.labels?.any?.length;
|
13
|
+
for (const label of options.labels?.any || []) {
|
14
|
+
if (labels.includes(label)) {
|
15
|
+
matched++;
|
16
|
+
foundAny = true;
|
17
|
+
}
|
18
|
+
}
|
19
|
+
if (!foundAny)
|
20
|
+
return 0;
|
21
|
+
|
22
|
+
for (const label of options?.labels?.prefer || []) {
|
23
|
+
if (labels.includes(label))
|
24
|
+
matched++;
|
25
|
+
}
|
26
|
+
// ensure non zero result.
|
27
|
+
matched++;
|
28
|
+
return matched;
|
29
|
+
}
|
30
|
+
|
31
|
+
export function getClusterLabels() {
|
32
|
+
let labels = process.env.SCRYPTED_CLUSTER_LABELS?.split(',') || [];
|
33
|
+
labels.push(process.arch, process.platform, os.hostname());
|
34
|
+
labels = [...new Set(labels)];
|
35
|
+
return labels;
|
36
|
+
}
|
@@ -1,16 +1,15 @@
|
|
1
1
|
import { once } from 'events';
|
2
2
|
import net from 'net';
|
3
3
|
import worker_threads from 'worker_threads';
|
4
|
-
import {
|
5
|
-
import {
|
6
|
-
import {
|
7
|
-
import {
|
8
|
-
import {
|
9
|
-
import {
|
10
|
-
import {
|
11
|
-
|
12
|
-
|
13
|
-
export function getClusterPeerKey(address: string, port: number) {
|
4
|
+
import { Deferred } from '../deferred';
|
5
|
+
import { listenZero } from '../listen-zero';
|
6
|
+
import { NodeThreadWorker } from '../plugin/runtime/node-thread-worker';
|
7
|
+
import { RpcPeer } from "../rpc";
|
8
|
+
import { createDuplexRpcPeer } from "../rpc-serializer";
|
9
|
+
import { computeClusterObjectHash } from "./cluster-hash";
|
10
|
+
import { ClusterObject, ConnectRPCObject } from "./connect-rpc-object";
|
11
|
+
|
12
|
+
function getClusterPeerKey(address: string, port: number) {
|
14
13
|
return `${address}:${port}`;
|
15
14
|
}
|
16
15
|
|
@@ -18,7 +17,7 @@ export function isClusterAddress(address: string) {
|
|
18
17
|
return !address || address === process.env.SCRYPTED_CLUSTER_ADDRESS;
|
19
18
|
}
|
20
19
|
|
21
|
-
|
20
|
+
async function peerConnectRPCObject(peer: RpcPeer, o: ClusterObject) {
|
22
21
|
let peerConnectRPCObject: Promise<ConnectRPCObject> = peer.tags['connectRPCObject'];
|
23
22
|
if (!peerConnectRPCObject) {
|
24
23
|
peerConnectRPCObject = peer.getParam('connectRPCObject');
|
@@ -28,7 +27,7 @@ export async function peerConnectRPCObject(peer: RpcPeer, o: ClusterObject) {
|
|
28
27
|
return resolved(o);
|
29
28
|
}
|
30
29
|
|
31
|
-
export function
|
30
|
+
export function setupCluster(peer: RpcPeer) {
|
32
31
|
const SCRYPTED_CLUSTER_ADDRESS = process.env.SCRYPTED_CLUSTER_ADDRESS;
|
33
32
|
let clusterId: string;
|
34
33
|
let clusterSecret: string;
|
@@ -371,4 +370,57 @@ export function prepareClusterPeer(peer: RpcPeer) {
|
|
371
370
|
mainThreadBrokerRegister,
|
372
371
|
connectRPCObject,
|
373
372
|
}
|
374
|
-
}
|
373
|
+
}
|
374
|
+
|
375
|
+
export type InitializeCluster = (cluster: { clusterId: string, clusterSecret: string }) => Promise<void>;
|
376
|
+
|
377
|
+
export function getScryptedClusterMode(): ['server' | 'client', string, number] {
|
378
|
+
const mode = process.env.SCRYPTED_CLUSTER_MODE as 'server' | 'client';
|
379
|
+
|
380
|
+
if (!mode) {
|
381
|
+
if (process.env.SCRYPTED_CLUSTER_ADDRESS) {
|
382
|
+
console.warn('SCRYPTED_CLUSTER_ADDRESS, but SCRYPTED_CLUSTER_MODE is not set. This setting will be ignored.');
|
383
|
+
delete process.env.SCRYPTED_CLUSTER_ADDRESS;
|
384
|
+
}
|
385
|
+
if (process.env.SCRPYTED_CLUSTER_SERVER) {
|
386
|
+
console.warn('SCRYPTED_CLUSTER_SERVER, but SCRYPTED_CLUSTER_MODE is not set. This setting will be ignored.');
|
387
|
+
delete process.env.SCRPYTED_CLUSTER_SERVER
|
388
|
+
}
|
389
|
+
if (process.env.SCRYPTED_CLUSTER_SECRET) {
|
390
|
+
console.warn('SCRYPTED_CLUSTER_SECRET, but SCRYPTED_CLUSTER_MODE is not set. This setting will be ignored.');
|
391
|
+
delete process.env.SCRYPTED_CLUSTER_SECRET;
|
392
|
+
}
|
393
|
+
return;
|
394
|
+
}
|
395
|
+
|
396
|
+
if (!['server', 'client'].includes(mode))
|
397
|
+
throw new Error('SCRYPTED_CLUSTER_MODE must be set to either "server" or "client".');
|
398
|
+
|
399
|
+
if (!process.env.SCRYPTED_CLUSTER_SECRET)
|
400
|
+
throw new Error('SCRYPTED_CLUSTER_MODE is set but SCRYPTED_CLUSTER_SECRET is not set.');
|
401
|
+
|
402
|
+
const [server, sport] = process.env.SCRYPTED_CLUSTER_SERVER?.split(':') || [];
|
403
|
+
const port = parseInt(sport) || 10556;
|
404
|
+
const address = process.env.SCRYPTED_CLUSTER_ADDRESS;
|
405
|
+
|
406
|
+
if (mode === 'client') {
|
407
|
+
if (!net.isIP(server))
|
408
|
+
throw new Error('SCRYPTED_CLUSTER_SERVER is not a valid IP address:port.');
|
409
|
+
|
410
|
+
if (!net.isIP(address))
|
411
|
+
throw new Error('SCRYPTED_CLUSTER_ADDRESS is not set to a valid IP address.');
|
412
|
+
}
|
413
|
+
else {
|
414
|
+
// the cluster address may come from the server:port combo or address variable but not both.
|
415
|
+
if (address && server && server !== address)
|
416
|
+
throw new Error('SCRYPTED_CLUSTER_ADDRESS and SCRYPTED_CLUSTER_SERVER must not both be used.');
|
417
|
+
|
418
|
+
const serverAddress = address || server;
|
419
|
+
if (!net.isIP(serverAddress))
|
420
|
+
throw new Error('SCRYPTED_CLUSTER_ADDRESS is not set.');
|
421
|
+
process.env.SCRYPTED_CLUSTER_ADDRESS = serverAddress;
|
422
|
+
delete process.env.SCRYPTED_CLUSTER_SERVER;
|
423
|
+
}
|
424
|
+
|
425
|
+
return [mode, server, port];
|
426
|
+
}
|
@@ -4,7 +4,7 @@ import { once } from 'events';
|
|
4
4
|
import net, { Server } from 'net';
|
5
5
|
import { PassThrough, Readable, Writable } from 'stream';
|
6
6
|
import { listenZero } from '../listen-zero';
|
7
|
-
import { isClusterAddress } from '../
|
7
|
+
import { isClusterAddress } from '../cluster/cluster-setup';
|
8
8
|
|
9
9
|
export interface ConsoleServer {
|
10
10
|
pluginConsole: Console;
|
@@ -10,7 +10,7 @@ import { Logger } from '../logger';
|
|
10
10
|
import { RpcPeer, RPCResultError } from '../rpc';
|
11
11
|
import { createRpcSerializer } from '../rpc-serializer';
|
12
12
|
import { ScryptedRuntime } from '../runtime';
|
13
|
-
import {
|
13
|
+
import { setupCluster } from '../cluster/cluster-setup';
|
14
14
|
import { sleep } from '../sleep';
|
15
15
|
import { AccessControls } from './acl';
|
16
16
|
import { MediaManagerHostImpl } from './media';
|
@@ -342,7 +342,7 @@ export class PluginHost {
|
|
342
342
|
}
|
343
343
|
});
|
344
344
|
|
345
|
-
const clusterSetup =
|
345
|
+
const clusterSetup = setupCluster(this.peer);
|
346
346
|
const { runtimeWorker, forkPeer } = createClusterForkWorker((async () => {
|
347
347
|
await clusterSetup.initializeCluster({
|
348
348
|
clusterId: this.scrypted.clusterId,
|
@@ -394,12 +394,12 @@ export class PluginHost {
|
|
394
394
|
};
|
395
395
|
(async () => {
|
396
396
|
try {
|
397
|
-
let pingPromise: Promise<
|
397
|
+
let pingPromise: Promise<(time: number) => Promise<void>>
|
398
398
|
while (!this.killed) {
|
399
399
|
await sleep(30000);
|
400
400
|
if (this.killed)
|
401
401
|
return;
|
402
|
-
pingPromise ||= peer.
|
402
|
+
pingPromise ||= this.peer.getParam('ping');
|
403
403
|
const ping = await pingPromise;
|
404
404
|
await ping(Date.now());
|
405
405
|
}
|
@@ -1,17 +1,12 @@
|
|
1
1
|
import { ForkWorker, ScryptedStatic, SystemManager } from '@scrypted/types';
|
2
2
|
import child_process from 'child_process';
|
3
|
-
import { once } from 'events';
|
4
3
|
import fs from 'fs';
|
5
|
-
import net from 'net';
|
6
4
|
import path from 'path';
|
7
5
|
import { install as installSourceMapSupport } from 'source-map-support';
|
8
6
|
import worker_threads from 'worker_threads';
|
9
|
-
import {
|
10
|
-
import { Deferred } from '../deferred';
|
7
|
+
import { setupCluster } from '../cluster/cluster-setup';
|
11
8
|
import { RpcMessage, RpcPeer } from '../rpc';
|
12
9
|
import { evalLocal } from '../rpc-peer-eval';
|
13
|
-
import { createDuplexRpcPeer } from '../rpc-serializer';
|
14
|
-
import { getClusterPeerKey, isClusterAddress, peerConnectRPCObject, prepareClusterPeer } from '../scrypted-cluster-common';
|
15
10
|
import { MediaManagerImpl } from './media';
|
16
11
|
import { PluginAPI, PluginAPIProxy, PluginRemote, PluginRemoteLoadZipOptions, PluginZipAPI } from './plugin-api';
|
17
12
|
import { pipeWorkerConsole, prepareConsoles } from './plugin-console';
|
@@ -37,7 +32,7 @@ export interface StartPluginRemoteOptions {
|
|
37
32
|
export function startPluginRemote(mainFilename: string, pluginId: string, peerSend: (message: RpcMessage, reject?: (e: Error) => void, serializationContext?: any) => void, startPluginRemoteOptions?: StartPluginRemoteOptions) {
|
38
33
|
const peer = new RpcPeer('unknown', 'host', peerSend);
|
39
34
|
|
40
|
-
const clusterPeerSetup =
|
35
|
+
const clusterPeerSetup = setupCluster(peer);
|
41
36
|
const { initializeCluster, connectRPCObject, mainThreadBrokerRegister , mainThreadPort } = clusterPeerSetup;
|
42
37
|
|
43
38
|
peer.params.initializeCluster = initializeCluster;
|
@@ -1,11 +1,11 @@
|
|
1
1
|
import { EventEmitter, PassThrough } from "stream";
|
2
2
|
import { Deferred } from "../../deferred";
|
3
3
|
import { RpcPeer } from "../../rpc";
|
4
|
-
import {
|
4
|
+
import { getClusterLabels, matchesClusterLabels } from "../../cluster/cluster-labels";
|
5
|
+
import { ClusterForkOptions, PeerLiveness } from "../../scrypted-cluster-main";
|
5
6
|
import type { ClusterFork } from "../../services/cluster-fork";
|
6
7
|
import { writeWorkerGenerator } from "../plugin-console";
|
7
8
|
import type { RuntimeWorker } from "./runtime-worker";
|
8
|
-
import { sleep } from "../../sleep";
|
9
9
|
|
10
10
|
export function needsClusterForkWorker(options: ClusterForkOptions) {
|
11
11
|
return process.env.SCRYPTED_CLUSTER_ADDRESS && options?.runtime && !matchesClusterLabels(options, getClusterLabels())
|
@@ -4,7 +4,7 @@ import { RpcMessage, RpcPeer } from "../../rpc";
|
|
4
4
|
import { SidebandSocketSerializer } from "../socket-serializer";
|
5
5
|
import { ChildProcessWorker } from "./child-process-worker";
|
6
6
|
import { RuntimeWorkerOptions } from "./runtime-worker";
|
7
|
-
import { getScryptedClusterMode } from '../../
|
7
|
+
import { getScryptedClusterMode } from '../../cluster/cluster-setup';
|
8
8
|
|
9
9
|
export const NODE_PLUGIN_CHILD_PROCESS = 'child';
|
10
10
|
export const NODE_PLUGIN_FORK_PROCESS = 'fork';
|
package/src/runtime.ts
CHANGED
@@ -33,7 +33,6 @@ import { isConnectionUpgrade, PluginHttp } from './plugin/plugin-http';
|
|
33
33
|
import { WebSocketConnection } from './plugin/plugin-remote-websocket';
|
34
34
|
import { getPluginVolume } from './plugin/plugin-volume';
|
35
35
|
import { getBuiltinRuntimeHosts } from './plugin/runtime/runtime-host';
|
36
|
-
import { ClusterWorker } from './scrypted-cluster';
|
37
36
|
import { getIpAddress, SCRYPTED_INSECURE_PORT, SCRYPTED_SECURE_PORT } from './server-settings';
|
38
37
|
import { AddressSettings } from './services/addresses';
|
39
38
|
import { Alerts } from './services/alerts';
|
@@ -45,7 +44,8 @@ import { getNpmPackageInfo, PluginComponent } from './services/plugin';
|
|
45
44
|
import { ServiceControl } from './services/service-control';
|
46
45
|
import { UsersService } from './services/users';
|
47
46
|
import { getState, ScryptedStateManager, setState } from './state';
|
48
|
-
import { isClusterAddress } from './
|
47
|
+
import { isClusterAddress } from './cluster/cluster-setup';
|
48
|
+
import { ClusterWorker } from './scrypted-cluster-main';
|
49
49
|
|
50
50
|
interface DeviceProxyPair {
|
51
51
|
handler: PluginDeviceProxyHandler;
|