@scrypted/server 0.123.1 → 0.123.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/ip.d.ts +2 -0
- package/dist/ip.js +5 -4
- package/dist/ip.js.map +1 -1
- package/dist/listen-zero.d.ts +5 -2
- package/dist/listen-zero.js +2 -1
- package/dist/listen-zero.js.map +1 -1
- package/dist/plugin/plugin-api.d.ts +9 -3
- package/dist/plugin/plugin-api.js +10 -1
- package/dist/plugin/plugin-api.js.map +1 -1
- package/dist/plugin/plugin-console.d.ts +2 -1
- package/dist/plugin/plugin-console.js +27 -6
- package/dist/plugin/plugin-console.js.map +1 -1
- package/dist/plugin/plugin-host-api.js +1 -1
- package/dist/plugin/plugin-host-api.js.map +1 -1
- package/dist/plugin/plugin-host.d.ts +4 -7
- package/dist/plugin/plugin-host.js +76 -62
- package/dist/plugin/plugin-host.js.map +1 -1
- package/dist/plugin/plugin-lazy-remote.d.ts +3 -3
- package/dist/plugin/plugin-lazy-remote.js +2 -2
- package/dist/plugin/plugin-lazy-remote.js.map +1 -1
- package/dist/plugin/plugin-remote-stats.d.ts +2 -2
- package/dist/plugin/plugin-remote-stats.js +4 -2
- package/dist/plugin/plugin-remote-stats.js.map +1 -1
- package/dist/plugin/plugin-remote-worker.d.ts +0 -1
- package/dist/plugin/plugin-remote-worker.js +77 -335
- package/dist/plugin/plugin-remote-worker.js.map +1 -1
- package/dist/plugin/plugin-remote.d.ts +3 -3
- package/dist/plugin/plugin-remote.js +2 -2
- package/dist/plugin/plugin-remote.js.map +1 -1
- package/dist/plugin/plugin-repl.js +2 -1
- package/dist/plugin/plugin-repl.js.map +1 -1
- package/dist/plugin/runtime/cluster-fork.worker.d.ts +9 -0
- package/dist/plugin/runtime/cluster-fork.worker.js +73 -0
- package/dist/plugin/runtime/cluster-fork.worker.js.map +1 -0
- package/dist/plugin/runtime/custom-worker.js +2 -2
- package/dist/plugin/runtime/custom-worker.js.map +1 -1
- package/dist/plugin/runtime/node-fork-worker.js +5 -3
- package/dist/plugin/runtime/node-fork-worker.js.map +1 -1
- package/dist/plugin/runtime/python-worker.js +2 -2
- package/dist/plugin/runtime/python-worker.js.map +1 -1
- package/dist/rpc.d.ts +1 -0
- package/dist/rpc.js +3 -2
- package/dist/rpc.js.map +1 -1
- package/dist/runtime.d.ts +4 -0
- package/dist/runtime.js +16 -2
- package/dist/runtime.js.map +1 -1
- package/dist/scrypted-cluster-common.d.ts +22 -0
- package/dist/scrypted-cluster-common.js +332 -0
- package/dist/scrypted-cluster-common.js.map +1 -0
- package/dist/scrypted-cluster-main.d.ts +2 -0
- package/dist/scrypted-cluster-main.js +12 -0
- package/dist/scrypted-cluster-main.js.map +1 -0
- package/dist/scrypted-cluster.d.ts +38 -0
- package/dist/scrypted-cluster.js +277 -0
- package/dist/scrypted-cluster.js.map +1 -0
- package/dist/scrypted-main-exports.js +20 -14
- package/dist/scrypted-main-exports.js.map +1 -1
- package/dist/scrypted-server-main.js +8 -15
- package/dist/scrypted-server-main.js.map +1 -1
- package/dist/server-settings.d.ts +1 -0
- package/dist/server-settings.js +2 -1
- package/dist/server-settings.js.map +1 -1
- package/dist/services/backup.js.map +1 -1
- package/dist/services/cluster-fork.d.ts +7 -0
- package/dist/services/cluster-fork.js +25 -0
- package/dist/services/cluster-fork.js.map +1 -0
- package/dist/services/plugin.d.ts +2 -7
- package/dist/services/plugin.js +2 -17
- package/dist/services/plugin.js.map +1 -1
- package/package.json +2 -2
- package/python/plugin_remote.py +150 -135
- package/python/rpc_reader.py +3 -19
- package/src/ip.ts +5 -4
- package/src/listen-zero.ts +3 -2
- package/src/plugin/plugin-api.ts +11 -3
- package/src/plugin/plugin-console.ts +29 -7
- package/src/plugin/plugin-host-api.ts +1 -1
- package/src/plugin/plugin-host.ts +92 -77
- package/src/plugin/plugin-lazy-remote.ts +4 -4
- package/src/plugin/plugin-remote-stats.ts +6 -4
- package/src/plugin/plugin-remote-worker.ts +91 -376
- package/src/plugin/plugin-remote.ts +5 -5
- package/src/plugin/plugin-repl.ts +2 -1
- package/src/plugin/runtime/cluster-fork.worker.ts +92 -0
- package/src/plugin/runtime/custom-worker.ts +2 -2
- package/src/plugin/runtime/node-fork-worker.ts +6 -3
- package/src/plugin/runtime/python-worker.ts +2 -2
- package/src/rpc.ts +3 -2
- package/src/runtime.ts +17 -2
- package/src/scrypted-cluster-common.ts +374 -0
- package/src/scrypted-cluster-main.ts +12 -0
- package/src/scrypted-cluster.ts +329 -0
- package/src/scrypted-main-exports.ts +22 -16
- package/src/scrypted-server-main.ts +15 -23
- package/src/server-settings.ts +1 -0
- package/src/services/backup.ts +0 -1
- package/src/services/cluster-fork.ts +22 -0
- package/src/services/plugin.ts +3 -21
@@ -0,0 +1,92 @@
|
|
1
|
+
import { EventEmitter, PassThrough } from "stream";
|
2
|
+
import { Deferred } from "../../deferred";
|
3
|
+
import { RpcPeer } from "../../rpc";
|
4
|
+
import { ClusterForkOptions, getClusterLabels, matchesClusterLabels, PeerLiveness } from "../../scrypted-cluster";
|
5
|
+
import type { ClusterFork } from "../../services/cluster-fork";
|
6
|
+
import { writeWorkerGenerator } from "../plugin-console";
|
7
|
+
import type { RuntimeWorker } from "./runtime-worker";
|
8
|
+
import { sleep } from "../../sleep";
|
9
|
+
|
10
|
+
export function needsClusterForkWorker(options: ClusterForkOptions) {
|
11
|
+
return process.env.SCRYPTED_CLUSTER_ADDRESS && options?.runtime && !matchesClusterLabels(options, getClusterLabels())
|
12
|
+
}
|
13
|
+
|
14
|
+
export function createClusterForkWorker(
|
15
|
+
forkComponentPromise: ClusterFork | Promise<ClusterFork>,
|
16
|
+
zipHash: string,
|
17
|
+
getZip: () => Promise<Buffer>,
|
18
|
+
options: ClusterForkOptions,
|
19
|
+
packageJson: any,
|
20
|
+
connectRPCObject: (o: any) => Promise<any>) {
|
21
|
+
const waitKilled = new Deferred<void>();
|
22
|
+
waitKilled.promise.finally(() => events.emit('exit'));
|
23
|
+
const events = new EventEmitter();
|
24
|
+
|
25
|
+
const stdout = new PassThrough();
|
26
|
+
const stderr = new PassThrough();
|
27
|
+
|
28
|
+
const runtimeWorker: RuntimeWorker = {
|
29
|
+
pid: 'cluster',
|
30
|
+
stdout,
|
31
|
+
stderr,
|
32
|
+
on: events.on.bind(events),
|
33
|
+
once: events.once.bind(events),
|
34
|
+
removeListener: events.removeListener.bind(events),
|
35
|
+
kill: () => {
|
36
|
+
waitKilled.resolve();
|
37
|
+
},
|
38
|
+
} as any;
|
39
|
+
|
40
|
+
waitKilled.promise.finally(() => {
|
41
|
+
runtimeWorker.pid = undefined;
|
42
|
+
});
|
43
|
+
|
44
|
+
const forkPeer = (async () => {
|
45
|
+
// need to ensure this happens on next tick to prevent unhandled promise rejection.
|
46
|
+
// await sleep(0);
|
47
|
+
const forkComponent = await forkComponentPromise;
|
48
|
+
const peerLiveness = new PeerLiveness(new Deferred().promise);
|
49
|
+
const clusterForkResult = await forkComponent.fork(peerLiveness, options, packageJson, zipHash, getZip);
|
50
|
+
waitKilled.promise.finally(() => {
|
51
|
+
runtimeWorker.pid = undefined;
|
52
|
+
clusterForkResult.kill();
|
53
|
+
});
|
54
|
+
clusterForkResult.waitKilled().catch(() => { })
|
55
|
+
.finally(() => {
|
56
|
+
waitKilled.resolve();
|
57
|
+
});
|
58
|
+
|
59
|
+
try {
|
60
|
+
const clusterGetRemote = await connectRPCObject(await clusterForkResult.getResult());
|
61
|
+
const {
|
62
|
+
stdout: stdoutGen,
|
63
|
+
stderr: stderrGen,
|
64
|
+
getRemote
|
65
|
+
} = await clusterGetRemote();
|
66
|
+
|
67
|
+
writeWorkerGenerator(stdoutGen, stdout).catch(() => { });
|
68
|
+
writeWorkerGenerator(stderrGen, stderr).catch(() => { });
|
69
|
+
|
70
|
+
const directGetRemote = await connectRPCObject(getRemote);
|
71
|
+
if (directGetRemote === getRemote)
|
72
|
+
throw new Error('cluster fork peer not direct connected');
|
73
|
+
const peer: RpcPeer = directGetRemote[RpcPeer.PROPERTY_PROXY_PEER];
|
74
|
+
if (!peer)
|
75
|
+
throw new Error('cluster fork peer undefined?');
|
76
|
+
return peer;
|
77
|
+
}
|
78
|
+
catch (e) {
|
79
|
+
clusterForkResult.kill();
|
80
|
+
throw e;
|
81
|
+
}
|
82
|
+
})();
|
83
|
+
|
84
|
+
forkPeer.catch(() => {
|
85
|
+
waitKilled.resolve();
|
86
|
+
});
|
87
|
+
|
88
|
+
return {
|
89
|
+
runtimeWorker,
|
90
|
+
forkPeer,
|
91
|
+
}
|
92
|
+
}
|
@@ -26,13 +26,13 @@ export class CustomRuntimeWorker extends ChildProcessWorker {
|
|
26
26
|
const opts: child_process.ForkOptions | child_process.SpawnOptions = {
|
27
27
|
// stdin, stdout, stderr, peer in, peer out
|
28
28
|
stdio: ['pipe', 'pipe', 'pipe', 'pipe', 'pipe', 'pipe'],
|
29
|
-
env: Object.assign({
|
29
|
+
env: Object.assign({}, process.env, env, {
|
30
30
|
SCRYYPTED_PLUGIN_ID: pluginId,
|
31
31
|
SCRYPTED_DEBUG_PORT: pluginDebug?.inspectPort?.toString(),
|
32
32
|
SCRYPTED_UNZIPPED_PATH: options.unzippedPath,
|
33
33
|
SCRYPTED_ZIP_FILE: options.zipFile,
|
34
34
|
SCRYPTED_ZIP_HASH: options.zipHash,
|
35
|
-
}
|
35
|
+
}),
|
36
36
|
};
|
37
37
|
|
38
38
|
if (!scryptedRuntimeArguments.executable) {
|
@@ -4,6 +4,7 @@ import { RpcMessage, RpcPeer } from "../../rpc";
|
|
4
4
|
import { SidebandSocketSerializer } from "../socket-serializer";
|
5
5
|
import { ChildProcessWorker } from "./child-process-worker";
|
6
6
|
import { RuntimeWorkerOptions } from "./runtime-worker";
|
7
|
+
import { getScryptedClusterMode } from '../../scrypted-cluster';
|
7
8
|
|
8
9
|
export const NODE_PLUGIN_CHILD_PROCESS = 'child';
|
9
10
|
export const NODE_PLUGIN_FORK_PROCESS = 'fork';
|
@@ -39,12 +40,14 @@ export class NodeForkWorker extends ChildProcessWorker {
|
|
39
40
|
execArgv.push(`--inspect=0.0.0.0:${pluginDebug.inspectPort}`);
|
40
41
|
}
|
41
42
|
|
42
|
-
|
43
|
+
const args = [
|
43
44
|
// change the argument marker depending on whether this is the main scrypted server process
|
44
45
|
// starting a plugin vs the plugin forking for multiprocessing.
|
45
|
-
isNodePluginWorkerProcess() ? NODE_PLUGIN_FORK_PROCESS : NODE_PLUGIN_CHILD_PROCESS,
|
46
|
+
isNodePluginWorkerProcess() || getScryptedClusterMode()?.[0] === 'client' ? NODE_PLUGIN_FORK_PROCESS : NODE_PLUGIN_CHILD_PROCESS,
|
46
47
|
this.pluginId
|
47
|
-
]
|
48
|
+
];
|
49
|
+
|
50
|
+
this.worker = child_process.fork(mainFilename, args, {
|
48
51
|
stdio: ['pipe', 'pipe', 'pipe', 'ipc'],
|
49
52
|
env: Object.assign({}, process.env, env),
|
50
53
|
serialization: 'advanced',
|
@@ -100,12 +100,12 @@ export class PythonRuntimeWorker extends ChildProcessWorker {
|
|
100
100
|
cwd: options.unzippedPath,
|
101
101
|
// stdin, stdout, stderr, peer in, peer out
|
102
102
|
stdio: ['pipe', 'pipe', 'pipe', 'pipe', 'pipe'],
|
103
|
-
env: Object.assign({
|
103
|
+
env: Object.assign({}, process.env, env, gstEnv, {
|
104
104
|
// rev this if the base python version or server characteristics change.
|
105
105
|
SCRYPTED_PYTHON_VERSION: '20240317',
|
106
106
|
PYTHONUNBUFFERED: '1',
|
107
107
|
PYTHONPATH,
|
108
|
-
}
|
108
|
+
}),
|
109
109
|
});
|
110
110
|
this.setupWorker();
|
111
111
|
|
package/src/rpc.ts
CHANGED
@@ -117,7 +117,7 @@ class RpcProxy implements PrimitiveProxyHandler<any> {
|
|
117
117
|
return this.entry.id;
|
118
118
|
if (p === '__proxy_constructor')
|
119
119
|
return this.constructorName;
|
120
|
-
if (p ===
|
120
|
+
if (p === RpcPeer.PROPERTY_PROXY_PEER)
|
121
121
|
return this.peer;
|
122
122
|
if (p === RpcPeer.PROPERTY_PROXY_PROPERTIES)
|
123
123
|
return this.proxyProps;
|
@@ -374,6 +374,7 @@ export class RpcPeer {
|
|
374
374
|
static readonly RANDOM_DIGITS = 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789';
|
375
375
|
static readonly RPC_RESULT_ERROR_NAME = 'RPCResultError';
|
376
376
|
static readonly PROPERTY_PROXY_ID = '__proxy_id';
|
377
|
+
static readonly PROPERTY_PROXY_PEER = '__proxy_peer';
|
377
378
|
static readonly PROPERTY_PROXY_ONEWAY_METHODS = '__proxy_oneway_methods';
|
378
379
|
static readonly PROPERTY_JSON_DISABLE_SERIALIZATION = '__json_disable_serialization';
|
379
380
|
static readonly PROPERTY_PROXY_PROPERTIES = '__proxy_props';
|
@@ -383,7 +384,7 @@ export class RpcPeer {
|
|
383
384
|
'constructor',
|
384
385
|
'__proxy_id',
|
385
386
|
'__proxy_constructor',
|
386
|
-
|
387
|
+
RpcPeer.PROPERTY_PROXY_PEER,
|
387
388
|
RpcPeer.PROPERTY_PROXY_ONEWAY_METHODS,
|
388
389
|
RpcPeer.PROPERTY_JSON_DISABLE_SERIALIZATION,
|
389
390
|
RpcPeer.PROPERTY_PROXY_PROPERTIES,
|
package/src/runtime.ts
CHANGED
@@ -33,16 +33,19 @@ import { isConnectionUpgrade, PluginHttp } from './plugin/plugin-http';
|
|
33
33
|
import { WebSocketConnection } from './plugin/plugin-remote-websocket';
|
34
34
|
import { getPluginVolume } from './plugin/plugin-volume';
|
35
35
|
import { getBuiltinRuntimeHosts } from './plugin/runtime/runtime-host';
|
36
|
+
import { ClusterWorker } from './scrypted-cluster';
|
36
37
|
import { getIpAddress, SCRYPTED_INSECURE_PORT, SCRYPTED_SECURE_PORT } from './server-settings';
|
37
38
|
import { AddressSettings } from './services/addresses';
|
38
39
|
import { Alerts } from './services/alerts';
|
39
40
|
import { Backup } from './services/backup';
|
41
|
+
import { ClusterFork } from './services/cluster-fork';
|
40
42
|
import { CORSControl } from './services/cors';
|
41
43
|
import { Info } from './services/info';
|
42
44
|
import { getNpmPackageInfo, PluginComponent } from './services/plugin';
|
43
45
|
import { ServiceControl } from './services/service-control';
|
44
46
|
import { UsersService } from './services/users';
|
45
47
|
import { getState, ScryptedStateManager, setState } from './state';
|
48
|
+
import { isClusterAddress } from './scrypted-cluster-common';
|
46
49
|
|
47
50
|
interface DeviceProxyPair {
|
48
51
|
handler: PluginDeviceProxyHandler;
|
@@ -59,7 +62,8 @@ interface HttpPluginData {
|
|
59
62
|
|
60
63
|
export class ScryptedRuntime extends PluginHttp<HttpPluginData> {
|
61
64
|
clusterId = crypto.randomBytes(3).toString('hex');
|
62
|
-
clusterSecret = crypto.randomBytes(16).toString('hex');
|
65
|
+
clusterSecret = process.env.SCRYPTED_CLUSTER_SECRET || crypto.randomBytes(16).toString('hex');
|
66
|
+
clusterWorkers = new Set<ClusterWorker>();
|
63
67
|
plugins: { [id: string]: PluginHost } = {};
|
64
68
|
pluginDevices: { [id: string]: PluginDevice } = {};
|
65
69
|
devices: { [id: string]: DeviceProxyPair } = {};
|
@@ -85,6 +89,7 @@ export class ScryptedRuntime extends PluginHttp<HttpPluginData> {
|
|
85
89
|
corsControl = new CORSControl(this);
|
86
90
|
addressSettings = new AddressSettings(this);
|
87
91
|
usersService = new UsersService(this);
|
92
|
+
clusterFork = new ClusterFork(this);
|
88
93
|
info = new Info();
|
89
94
|
backup = new Backup(this);
|
90
95
|
pluginHosts = getBuiltinRuntimeHosts();
|
@@ -119,7 +124,13 @@ export class ScryptedRuntime extends PluginHttp<HttpPluginData> {
|
|
119
124
|
return;
|
120
125
|
}
|
121
126
|
|
122
|
-
|
127
|
+
let address = clusterObject.address;
|
128
|
+
if (isClusterAddress(address))
|
129
|
+
address = '127.0.0.1';
|
130
|
+
const socket = net.connect({
|
131
|
+
port: clusterObject.port,
|
132
|
+
host: address,
|
133
|
+
});
|
123
134
|
socket.on('error', () => connection.close());
|
124
135
|
socket.on('close', () => connection.close());
|
125
136
|
socket.on('data', data => connection.send(data));
|
@@ -220,6 +231,8 @@ export class ScryptedRuntime extends PluginHttp<HttpPluginData> {
|
|
220
231
|
}
|
221
232
|
|
222
233
|
getDeviceLogger(device: PluginDevice): Logger {
|
234
|
+
if (!device)
|
235
|
+
return;
|
223
236
|
return this.devicesLogger.getLogger(device._id, getState(device, ScryptedInterfaceProperty.name));
|
224
237
|
}
|
225
238
|
|
@@ -361,6 +374,8 @@ export class ScryptedRuntime extends PluginHttp<HttpPluginData> {
|
|
361
374
|
return this.usersService;
|
362
375
|
case 'backup':
|
363
376
|
return this.backup;
|
377
|
+
case 'cluster-fork':
|
378
|
+
return this.clusterFork;
|
364
379
|
}
|
365
380
|
}
|
366
381
|
|
@@ -0,0 +1,374 @@
|
|
1
|
+
import { once } from 'events';
|
2
|
+
import net from 'net';
|
3
|
+
import worker_threads from 'worker_threads';
|
4
|
+
import { computeClusterObjectHash } from "./cluster/cluster-hash";
|
5
|
+
import { ClusterObject, ConnectRPCObject } from "./cluster/connect-rpc-object";
|
6
|
+
import { Deferred } from './deferred';
|
7
|
+
import { listenZero } from './listen-zero';
|
8
|
+
import { NodeThreadWorker } from './plugin/runtime/node-thread-worker';
|
9
|
+
import { RpcPeer } from "./rpc";
|
10
|
+
import { createDuplexRpcPeer } from "./rpc-serializer";
|
11
|
+
import { InitializeCluster } from "./scrypted-cluster";
|
12
|
+
|
13
|
+
export function getClusterPeerKey(address: string, port: number) {
|
14
|
+
return `${address}:${port}`;
|
15
|
+
}
|
16
|
+
|
17
|
+
export function isClusterAddress(address: string) {
|
18
|
+
return !address || address === process.env.SCRYPTED_CLUSTER_ADDRESS;
|
19
|
+
}
|
20
|
+
|
21
|
+
export async function peerConnectRPCObject(peer: RpcPeer, o: ClusterObject) {
|
22
|
+
let peerConnectRPCObject: Promise<ConnectRPCObject> = peer.tags['connectRPCObject'];
|
23
|
+
if (!peerConnectRPCObject) {
|
24
|
+
peerConnectRPCObject = peer.getParam('connectRPCObject');
|
25
|
+
peer.tags['connectRPCObject'] = peerConnectRPCObject;
|
26
|
+
}
|
27
|
+
const resolved = await peerConnectRPCObject;
|
28
|
+
return resolved(o);
|
29
|
+
}
|
30
|
+
|
31
|
+
export function prepareClusterPeer(peer: RpcPeer) {
|
32
|
+
const SCRYPTED_CLUSTER_ADDRESS = process.env.SCRYPTED_CLUSTER_ADDRESS;
|
33
|
+
let clusterId: string;
|
34
|
+
let clusterSecret: string;
|
35
|
+
let clusterPort: number;
|
36
|
+
|
37
|
+
// all cluster clients, incoming and outgoing, connect with random ports which can be used as peer ids
|
38
|
+
// on the cluster server that is listening on the actual port.
|
39
|
+
// incoming connections: use the remote random/unique port
|
40
|
+
// outgoing connections: use the local random/unique port
|
41
|
+
const clusterPeers = new Map<string, Promise<RpcPeer>>();
|
42
|
+
|
43
|
+
const resolveObject = async (id: string, sourceKey: string) => {
|
44
|
+
const sourcePeer = sourceKey
|
45
|
+
? await clusterPeers.get(sourceKey)
|
46
|
+
: peer;
|
47
|
+
if (!sourcePeer)
|
48
|
+
console.error('source peer not found', sourceKey);
|
49
|
+
const ret = sourcePeer?.localProxyMap.get(id);
|
50
|
+
if (!ret) {
|
51
|
+
console.error('source key not found', sourceKey, id);
|
52
|
+
return;
|
53
|
+
}
|
54
|
+
return ret;
|
55
|
+
}
|
56
|
+
|
57
|
+
const connectClusterObject = async (o: ClusterObject) => {
|
58
|
+
const sha256 = computeClusterObjectHash(o, clusterSecret);
|
59
|
+
if (sha256 !== o.sha256)
|
60
|
+
throw new Error('secret incorrect');
|
61
|
+
return resolveObject(o.proxyId, o.sourceKey);
|
62
|
+
}
|
63
|
+
|
64
|
+
const ensureClusterPeer = (address: string, connectPort: number) => {
|
65
|
+
if (isClusterAddress(address))
|
66
|
+
address = '127.0.0.1';
|
67
|
+
|
68
|
+
const clusterPeerKey = getClusterPeerKey(address, connectPort);
|
69
|
+
let clusterPeerPromise = clusterPeers.get(clusterPeerKey);
|
70
|
+
if (clusterPeerPromise)
|
71
|
+
return clusterPeerPromise;
|
72
|
+
|
73
|
+
clusterPeerPromise = (async () => {
|
74
|
+
const socket = net.connect(connectPort, address);
|
75
|
+
socket.on('close', () => clusterPeers.delete(clusterPeerKey));
|
76
|
+
|
77
|
+
try {
|
78
|
+
await once(socket, 'connect');
|
79
|
+
const { address: sourceAddress } = (socket.address() as net.AddressInfo);
|
80
|
+
if (sourceAddress !== SCRYPTED_CLUSTER_ADDRESS && sourceAddress !== '127.0.0.1') {
|
81
|
+
// source address may end with .0 if its a gateway into docker.
|
82
|
+
if (!sourceAddress.endsWith('.0'))
|
83
|
+
console.warn("source address mismatch", sourceAddress);
|
84
|
+
}
|
85
|
+
|
86
|
+
const clusterPeer = createDuplexRpcPeer(peer.selfName, clusterPeerKey, socket, socket);
|
87
|
+
// set params from the primary peer, needed for get getRemote in cluster mode.
|
88
|
+
Object.assign(clusterPeer.params, peer.params);
|
89
|
+
clusterPeer.onProxySerialization = (value) => onProxySerialization(clusterPeer, value, clusterPeerKey);
|
90
|
+
return clusterPeer;
|
91
|
+
}
|
92
|
+
catch (e) {
|
93
|
+
console.error('failure ipc connect', e);
|
94
|
+
socket.destroy();
|
95
|
+
throw e;
|
96
|
+
}
|
97
|
+
})();
|
98
|
+
|
99
|
+
clusterPeers.set(clusterPeerKey, clusterPeerPromise);
|
100
|
+
return clusterPeerPromise;
|
101
|
+
};
|
102
|
+
|
103
|
+
|
104
|
+
const tidChannels = new Map<number, Deferred<worker_threads.MessagePort>>();
|
105
|
+
const tidPeers = new Map<number, Promise<RpcPeer>>();
|
106
|
+
|
107
|
+
function connectTidPeer(tid: number) {
|
108
|
+
let peerPromise = tidPeers.get(tid);
|
109
|
+
if (peerPromise)
|
110
|
+
return peerPromise;
|
111
|
+
let tidDeferred = tidChannels.get(tid);
|
112
|
+
// if the tid port is not available yet, request it.
|
113
|
+
if (!tidDeferred) {
|
114
|
+
tidDeferred = new Deferred<worker_threads.MessagePort>();
|
115
|
+
tidChannels.set(tid, tidDeferred);
|
116
|
+
|
117
|
+
if (mainThreadPort) {
|
118
|
+
// request the connection via the main thread
|
119
|
+
mainThreadPort.postMessage({
|
120
|
+
threadId: tid,
|
121
|
+
});
|
122
|
+
}
|
123
|
+
}
|
124
|
+
|
125
|
+
const threadPeerKey = `thread:${tid}`;
|
126
|
+
function peerCleanup() {
|
127
|
+
clusterPeers.delete(threadPeerKey);
|
128
|
+
}
|
129
|
+
peerPromise = tidDeferred.promise.then(port => {
|
130
|
+
const threadPeer = NodeThreadWorker.createRpcPeer(peer.selfName, threadPeerKey, port);
|
131
|
+
// set params from the primary peer, needed for get getRemote in cluster mode.
|
132
|
+
Object.assign(threadPeer.params, peer.params);
|
133
|
+
threadPeer.onProxySerialization = value => onProxySerialization(threadPeer, value, threadPeerKey);
|
134
|
+
|
135
|
+
threadPeer.params.connectRPCObject = connectClusterObject;
|
136
|
+
|
137
|
+
function cleanup(message: string) {
|
138
|
+
peerCleanup();
|
139
|
+
tidChannels.delete(tid);
|
140
|
+
tidPeers.delete(tid);
|
141
|
+
threadPeer.kill(message);
|
142
|
+
}
|
143
|
+
port.on('close', () => cleanup('connection closed.'));
|
144
|
+
port.on('messageerror', () => cleanup('message error.'));
|
145
|
+
return threadPeer;
|
146
|
+
});
|
147
|
+
peerPromise.catch(() => peerCleanup());
|
148
|
+
clusterPeers.set(threadPeerKey, peerPromise);
|
149
|
+
tidPeers.set(tid, peerPromise);
|
150
|
+
|
151
|
+
return peerPromise;
|
152
|
+
}
|
153
|
+
|
154
|
+
const mainThreadPort: worker_threads.MessagePort = worker_threads.isMainThread ? undefined : worker_threads.workerData.mainThreadPort;
|
155
|
+
if (!worker_threads.isMainThread) {
|
156
|
+
// the main thread port will send messages with a thread port when a thread wants to initiate a connection.
|
157
|
+
mainThreadPort.on('message', async (message: { port: worker_threads.MessagePort, threadId: number }) => {
|
158
|
+
const { port, threadId } = message;
|
159
|
+
let tidDeferred = tidChannels.get(threadId);
|
160
|
+
if (!tidDeferred) {
|
161
|
+
tidDeferred = new Deferred<worker_threads.MessagePort>();
|
162
|
+
tidChannels.set(threadId, tidDeferred);
|
163
|
+
}
|
164
|
+
tidDeferred.resolve(port);
|
165
|
+
connectTidPeer(threadId);
|
166
|
+
});
|
167
|
+
}
|
168
|
+
|
169
|
+
async function connectIPCObject(clusterObject: ClusterObject, tid: number) {
|
170
|
+
// if the main thread is trying to connect to an object,
|
171
|
+
// the argument order matters here, as the connection attempt looks at the
|
172
|
+
// connectThreadId to see if the target is main thread.
|
173
|
+
if (worker_threads.isMainThread)
|
174
|
+
mainThreadBrokerConnect(tid, worker_threads.threadId);
|
175
|
+
const clusterPeer = await connectTidPeer(tid);
|
176
|
+
const existing = clusterPeer.remoteWeakProxies[clusterObject.proxyId]?.deref();
|
177
|
+
if (existing)
|
178
|
+
return existing;
|
179
|
+
return peerConnectRPCObject(clusterPeer, clusterObject);
|
180
|
+
}
|
181
|
+
|
182
|
+
const brokeredConnections = new Set<string>();
|
183
|
+
const workers = new Map<number, worker_threads.MessagePort>();
|
184
|
+
function mainThreadBrokerConnect(threadId: number, connectThreadId: number) {
|
185
|
+
if (worker_threads.isMainThread && threadId === worker_threads.threadId) {
|
186
|
+
const msg = 'invalid ipc, main thread cannot connect to itself';
|
187
|
+
console.error(msg);
|
188
|
+
throw new Error(msg);
|
189
|
+
}
|
190
|
+
// both workers nay initiate connection to each other at same time, so this
|
191
|
+
// is a synchronization point.
|
192
|
+
const key = JSON.stringify([threadId, connectThreadId].sort());
|
193
|
+
if (brokeredConnections.has(key))
|
194
|
+
return;
|
195
|
+
|
196
|
+
brokeredConnections.add(key);
|
197
|
+
|
198
|
+
const worker = workers.get(threadId);
|
199
|
+
const connect = workers.get(connectThreadId);
|
200
|
+
const channel = new worker_threads.MessageChannel();
|
201
|
+
|
202
|
+
worker.postMessage({
|
203
|
+
port: channel.port1,
|
204
|
+
threadId: connectThreadId,
|
205
|
+
}, [channel.port1]);
|
206
|
+
|
207
|
+
if (connect) {
|
208
|
+
connect.postMessage({
|
209
|
+
port: channel.port2,
|
210
|
+
threadId,
|
211
|
+
}, [channel.port2]);
|
212
|
+
}
|
213
|
+
else if (connectThreadId === worker_threads.threadId) {
|
214
|
+
connectTidPeer(threadId);
|
215
|
+
const deferred = tidChannels.get(threadId);
|
216
|
+
deferred.resolve(channel.port2);
|
217
|
+
}
|
218
|
+
else {
|
219
|
+
channel.port2.close();
|
220
|
+
}
|
221
|
+
}
|
222
|
+
|
223
|
+
function mainThreadBrokerRegister(workerPort: worker_threads.MessagePort, threadId: number) {
|
224
|
+
workers.set(threadId, workerPort);
|
225
|
+
|
226
|
+
// this is main thread, so there will be two types of requests from the child: registration requests from grandchildren and connection requests.
|
227
|
+
workerPort.on('message', async (message: { port: worker_threads.MessagePort, threadId: number }) => {
|
228
|
+
const { port, threadId: connectThreadId } = message;
|
229
|
+
|
230
|
+
if (port) {
|
231
|
+
mainThreadBrokerRegister(port, connectThreadId);
|
232
|
+
}
|
233
|
+
else {
|
234
|
+
mainThreadBrokerConnect(threadId, connectThreadId);
|
235
|
+
}
|
236
|
+
});
|
237
|
+
}
|
238
|
+
|
239
|
+
const connectRPCObject = async (value: any) => {
|
240
|
+
const clusterObject: ClusterObject = value?.__cluster;
|
241
|
+
if (clusterObject?.id !== clusterId)
|
242
|
+
return value;
|
243
|
+
const { address, port, proxyId } = clusterObject;
|
244
|
+
// handle the case when trying to connect to an object is on this cluster node,
|
245
|
+
// returning the actual object, rather than initiating a loopback connection.
|
246
|
+
if (port === clusterPort)
|
247
|
+
return connectClusterObject(clusterObject);
|
248
|
+
|
249
|
+
// can use worker to worker ipc if the address and pid matches and both side are node.
|
250
|
+
if (address === SCRYPTED_CLUSTER_ADDRESS && proxyId.startsWith('n-')) {
|
251
|
+
const parts = proxyId.split('-');
|
252
|
+
const pid = parseInt(parts[1]);
|
253
|
+
if (pid === process.pid)
|
254
|
+
return connectIPCObject(clusterObject, parseInt(parts[2]));
|
255
|
+
}
|
256
|
+
|
257
|
+
try {
|
258
|
+
const clusterPeerPromise = ensureClusterPeer(address, port);
|
259
|
+
const clusterPeer = await clusterPeerPromise;
|
260
|
+
// may already have this proxy so check first.
|
261
|
+
const existing = clusterPeer.remoteWeakProxies[proxyId]?.deref();
|
262
|
+
if (existing)
|
263
|
+
return existing;
|
264
|
+
const newValue = await peerConnectRPCObject(clusterPeer, clusterObject);
|
265
|
+
if (!newValue)
|
266
|
+
throw new Error('rpc object not found?');
|
267
|
+
return newValue;
|
268
|
+
}
|
269
|
+
catch (e) {
|
270
|
+
console.error('failure rpc', clusterObject, e);
|
271
|
+
return value;
|
272
|
+
}
|
273
|
+
}
|
274
|
+
|
275
|
+
const onProxySerialization = (peer: RpcPeer, value: any, sourceKey: string) => {
|
276
|
+
const properties = RpcPeer.prepareProxyProperties(value) || {};
|
277
|
+
let clusterEntry: ClusterObject = properties.__cluster;
|
278
|
+
|
279
|
+
// ensure globally stable proxyIds.
|
280
|
+
// worker threads will embed their pid and tid in the proxy id for cross worker fast path.
|
281
|
+
const proxyId = peer.localProxied.get(value)?.id || clusterEntry?.proxyId || `n-${process.pid}-${worker_threads.threadId}-${RpcPeer.generateId()}`;
|
282
|
+
|
283
|
+
// if the cluster entry already exists, check if it belongs to this node.
|
284
|
+
// if it belongs to this node, the entry must also be for this peer.
|
285
|
+
// relying on the liveness/gc of a different peer may cause race conditions.
|
286
|
+
if (clusterEntry) {
|
287
|
+
if (isClusterAddress(clusterEntry?.address) && clusterPort === clusterEntry.port && sourceKey !== clusterEntry.sourceKey)
|
288
|
+
clusterEntry = undefined;
|
289
|
+
}
|
290
|
+
|
291
|
+
if (!clusterEntry) {
|
292
|
+
clusterEntry = {
|
293
|
+
id: clusterId,
|
294
|
+
address: SCRYPTED_CLUSTER_ADDRESS,
|
295
|
+
port: clusterPort,
|
296
|
+
proxyId,
|
297
|
+
sourceKey,
|
298
|
+
sha256: null,
|
299
|
+
};
|
300
|
+
clusterEntry.sha256 = computeClusterObjectHash(clusterEntry, clusterSecret);
|
301
|
+
properties.__cluster = clusterEntry;
|
302
|
+
}
|
303
|
+
|
304
|
+
return {
|
305
|
+
proxyId,
|
306
|
+
properties,
|
307
|
+
};
|
308
|
+
}
|
309
|
+
const initializeCluster: InitializeCluster = async (options: {
|
310
|
+
clusterId: string;
|
311
|
+
clusterSecret: string;
|
312
|
+
}) => {
|
313
|
+
if (clusterPort)
|
314
|
+
return;
|
315
|
+
|
316
|
+
({ clusterId, clusterSecret } = options);
|
317
|
+
|
318
|
+
|
319
|
+
const clients = new Set<net.Socket>();
|
320
|
+
|
321
|
+
const clusterRpcServer = net.createServer(client => {
|
322
|
+
const clusterPeerAddress = client.remoteAddress;
|
323
|
+
const clusterPeerPort = client.remotePort;
|
324
|
+
const clusterPeerKey = getClusterPeerKey(clusterPeerAddress, clusterPeerPort);
|
325
|
+
const clusterPeer = createDuplexRpcPeer(peer.selfName, clusterPeerKey, client, client);
|
326
|
+
// set params from the primary peer, needed for get getRemote in cluster mode.
|
327
|
+
Object.assign(clusterPeer.params, peer.params);
|
328
|
+
// the listening peer sourceKey (client address/port) is used by the OTHER peer (the client)
|
329
|
+
// to determine if it is already connected to THIS peer (the server).
|
330
|
+
clusterPeer.onProxySerialization = (value) => onProxySerialization(clusterPeer, value, clusterPeerKey);
|
331
|
+
clusterPeers.set(clusterPeerKey, Promise.resolve(clusterPeer));
|
332
|
+
clusterPeer.params.connectRPCObject = connectClusterObject;
|
333
|
+
client.on('close', () => {
|
334
|
+
clusterPeers.delete(clusterPeerKey);
|
335
|
+
clusterPeer.kill('cluster socket closed');
|
336
|
+
clients.delete(client);
|
337
|
+
});
|
338
|
+
clients.add(client);
|
339
|
+
});
|
340
|
+
|
341
|
+
const listenAddress = SCRYPTED_CLUSTER_ADDRESS
|
342
|
+
? '0.0.0.0'
|
343
|
+
: '127.0.0.1';
|
344
|
+
|
345
|
+
clusterPort = await listenZero(clusterRpcServer, listenAddress);
|
346
|
+
peer.onProxySerialization = value => onProxySerialization(peer, value, undefined);
|
347
|
+
delete peer.params.initializeCluster;
|
348
|
+
|
349
|
+
peer.killed.catch(() => { }).finally(() => clusterRpcServer.close());
|
350
|
+
clusterRpcServer.on('close', () => {
|
351
|
+
peer.kill('cluster server closed');
|
352
|
+
// close all clusterRpcServer clients
|
353
|
+
for (const client of clients) {
|
354
|
+
client.destroy();
|
355
|
+
}
|
356
|
+
clients.clear();
|
357
|
+
});
|
358
|
+
}
|
359
|
+
|
360
|
+
return {
|
361
|
+
initializeCluster,
|
362
|
+
get clusterPort() {
|
363
|
+
return clusterPort;
|
364
|
+
},
|
365
|
+
SCRYPTED_CLUSTER_ADDRESS,
|
366
|
+
clusterPeers,
|
367
|
+
onProxySerialization,
|
368
|
+
connectClusterObject,
|
369
|
+
ensureClusterPeer,
|
370
|
+
mainThreadPort,
|
371
|
+
mainThreadBrokerRegister,
|
372
|
+
connectRPCObject,
|
373
|
+
}
|
374
|
+
}
|
@@ -0,0 +1,12 @@
|
|
1
|
+
import { install as installSourceMapSupport } from 'source-map-support';
|
2
|
+
import { startClusterClient } from './scrypted-cluster';
|
3
|
+
|
4
|
+
installSourceMapSupport({
|
5
|
+
environment: 'node',
|
6
|
+
});
|
7
|
+
|
8
|
+
async function start(mainFilename: string) {
|
9
|
+
startClusterClient(mainFilename);
|
10
|
+
}
|
11
|
+
|
12
|
+
export default start;
|