@scrypted/server 0.115.1 → 0.115.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/deno/deno-plugin-remote.ts +4 -0
- package/dist/cluster/cluster-hash.js +1 -1
- package/dist/cluster/cluster-hash.js.map +1 -1
- package/dist/cluster/connect-rpc-object.d.ts +2 -1
- package/dist/plugin/plugin-api.d.ts +1 -0
- package/dist/plugin/plugin-remote-stats.js +19 -15
- package/dist/plugin/plugin-remote-stats.js.map +1 -1
- package/dist/plugin/plugin-remote-worker.js +77 -45
- package/dist/plugin/plugin-remote-worker.js.map +1 -1
- package/dist/plugin/runtime/deno-worker.d.ts +12 -0
- package/dist/plugin/runtime/deno-worker.js +84 -0
- package/dist/plugin/runtime/deno-worker.js.map +1 -0
- package/dist/plugin/runtime/node-fork-worker.d.ts +1 -1
- package/dist/plugin/runtime/node-fork-worker.js +2 -2
- package/dist/plugin/runtime/node-fork-worker.js.map +1 -1
- package/dist/plugin/runtime/node-thread-worker.d.ts +3 -3
- package/dist/plugin/runtime/node-thread-worker.js +22 -7
- package/dist/plugin/runtime/node-thread-worker.js.map +1 -1
- package/dist/rpc-peer-eval.d.ts +4 -1
- package/dist/rpc-peer-eval.js +4 -10
- package/dist/rpc-peer-eval.js.map +1 -1
- package/dist/rpc.js +2 -2
- package/dist/rpc.js.map +1 -1
- package/dist/runtime.js +2 -0
- package/dist/runtime.js.map +1 -1
- package/dist/scrypted-main-exports.js +2 -2
- package/dist/scrypted-main-exports.js.map +1 -1
- package/dist/scrypted-main.js +4 -1
- package/dist/scrypted-main.js.map +1 -1
- package/dist/scrypted-plugin-main.js +14 -4
- package/dist/scrypted-plugin-main.js.map +1 -1
- package/package.json +4 -3
- package/python/plugin_remote.py +406 -218
- package/src/cluster/cluster-hash.ts +1 -1
- package/src/cluster/connect-rpc-object.ts +2 -1
- package/src/plugin/plugin-api.ts +1 -0
- package/src/plugin/plugin-remote-stats.ts +20 -15
- package/src/plugin/plugin-remote-worker.ts +87 -47
- package/src/plugin/runtime/deno-worker.ts +91 -0
- package/src/plugin/runtime/node-fork-worker.ts +3 -5
- package/src/plugin/runtime/node-thread-worker.ts +22 -6
- package/src/rpc-peer-eval.ts +9 -14
- package/src/rpc.ts +3 -3
- package/src/runtime.ts +2 -0
- package/src/scrypted-main-exports.ts +2 -2
- package/src/scrypted-main.ts +4 -1
- package/src/scrypted-plugin-main.ts +14 -4
@@ -2,6 +2,6 @@ import crypto from "crypto";
|
|
2
2
|
import { ClusterObject } from "./connect-rpc-object";
|
3
3
|
|
4
4
|
export function computeClusterObjectHash(o: ClusterObject, clusterSecret: string) {
|
5
|
-
const sha256 = crypto.createHash('sha256').update(`${o.id}${o.port}${o.
|
5
|
+
const sha256 = crypto.createHash('sha256').update(`${o.id}${o.address || ''}${o.port}${o.sourceKey || ''}${o.proxyId}${clusterSecret}`).digest().toString('base64');
|
6
6
|
return sha256;
|
7
7
|
}
|
package/src/plugin/plugin-api.ts
CHANGED
@@ -8,23 +8,28 @@ export interface PluginStats {
|
|
8
8
|
|
9
9
|
export function startStatsUpdater(allMemoryStats: Map<NodeThreadWorker, NodeJS.MemoryUsage>, updateStats: (stats: PluginStats) => void) {
|
10
10
|
setInterval(() => {
|
11
|
-
|
12
|
-
|
11
|
+
let cpuUsage: NodeJS.CpuUsage;
|
12
|
+
let memoryUsage: NodeJS.MemoryUsage;
|
13
|
+
if (process.cpuUsage) {
|
14
|
+
cpuUsage = process.cpuUsage();
|
15
|
+
allMemoryStats.set(undefined, process.memoryUsage());
|
13
16
|
|
14
|
-
|
15
|
-
|
16
|
-
|
17
|
-
|
18
|
-
|
19
|
-
|
20
|
-
|
17
|
+
memoryUsage = {
|
18
|
+
rss: 0,
|
19
|
+
heapTotal: 0,
|
20
|
+
heapUsed: 0,
|
21
|
+
external: 0,
|
22
|
+
arrayBuffers: 0,
|
23
|
+
}
|
24
|
+
|
25
|
+
for (const mu of allMemoryStats.values()) {
|
26
|
+
memoryUsage.rss += mu.rss;
|
27
|
+
memoryUsage.heapTotal += mu.heapTotal;
|
28
|
+
memoryUsage.heapUsed += mu.heapUsed;
|
29
|
+
memoryUsage.external += mu.external;
|
30
|
+
memoryUsage.arrayBuffers += mu.arrayBuffers;
|
31
|
+
}
|
21
32
|
|
22
|
-
for (const mu of allMemoryStats.values()) {
|
23
|
-
memoryUsage.rss += mu.rss;
|
24
|
-
memoryUsage.heapTotal += mu.heapTotal;
|
25
|
-
memoryUsage.heapUsed += mu.heapUsed;
|
26
|
-
memoryUsage.external += mu.external;
|
27
|
-
memoryUsage.arrayBuffers += mu.arrayBuffers;
|
28
33
|
}
|
29
34
|
|
30
35
|
updateStats({
|
@@ -93,10 +93,17 @@ export function startPluginRemote(mainFilename: string, pluginId: string, peerSe
|
|
93
93
|
throw new Error(`unknown service ${name}`);
|
94
94
|
},
|
95
95
|
async onLoadZip(scrypted: ScryptedStatic, params: any, packageJson: any, getZip: () => Promise<Buffer>, zipOptions: PluginRemoteLoadZipOptions) {
|
96
|
+
const mainFile = zipOptions?.main || 'main';
|
97
|
+
const mainNodejs = `${mainFile}.nodejs.js`;
|
98
|
+
const pluginMainNodeJs = `/plugin/${mainNodejs}`;
|
99
|
+
const pluginIdMainNodeJs = `/${pluginId}/${mainNodejs}`;
|
100
|
+
|
96
101
|
const { clusterId, clusterSecret, zipHash } = zipOptions;
|
97
102
|
const { zipFile, unzippedPath } = await prepareZip(getPluginVolume(pluginId), zipHash, getZip);
|
98
103
|
|
99
|
-
const
|
104
|
+
const SCRYPTED_CLUSTER_ADDRESS = process.env.SCRYPTED_CLUSTER_ADDRESS;
|
105
|
+
|
106
|
+
const onProxySerialization = (value: any, sourceKey?: string) => {
|
100
107
|
const properties = RpcPeer.prepareProxyProperties(value) || {};
|
101
108
|
let clusterEntry: ClusterObject = properties.__cluster;
|
102
109
|
|
@@ -106,16 +113,16 @@ export function startPluginRemote(mainFilename: string, pluginId: string, peerSe
|
|
106
113
|
// if the cluster entry already exists, check if it belongs to this node.
|
107
114
|
// if it belongs to this node, the entry must also be for this peer.
|
108
115
|
// relying on the liveness/gc of a different peer may cause race conditions.
|
109
|
-
if (clusterEntry && clusterPort === clusterEntry.port &&
|
116
|
+
if (clusterEntry && clusterPort === clusterEntry.port && sourceKey !== clusterEntry.sourceKey)
|
110
117
|
clusterEntry = undefined;
|
111
118
|
|
112
|
-
// set the cluster identity if it does not exist.
|
113
119
|
if (!clusterEntry) {
|
114
120
|
clusterEntry = {
|
115
121
|
id: clusterId,
|
122
|
+
address: SCRYPTED_CLUSTER_ADDRESS,
|
116
123
|
port: clusterPort,
|
117
124
|
proxyId,
|
118
|
-
|
125
|
+
sourceKey,
|
119
126
|
sha256: null,
|
120
127
|
};
|
121
128
|
clusterEntry.sha256 = computeClusterObjectHash(clusterEntry, clusterSecret);
|
@@ -129,8 +136,10 @@ export function startPluginRemote(mainFilename: string, pluginId: string, peerSe
|
|
129
136
|
}
|
130
137
|
peer.onProxySerialization = onProxySerialization;
|
131
138
|
|
132
|
-
const resolveObject = async (id: string,
|
133
|
-
const sourcePeer =
|
139
|
+
const resolveObject = async (id: string, sourceKey: string) => {
|
140
|
+
const sourcePeer = sourceKey
|
141
|
+
? await clusterPeers.get(sourceKey)
|
142
|
+
: peer;
|
134
143
|
return sourcePeer?.localProxyMap.get(id);
|
135
144
|
}
|
136
145
|
|
@@ -138,52 +147,74 @@ export function startPluginRemote(mainFilename: string, pluginId: string, peerSe
|
|
138
147
|
// on the cluster server that is listening on the actual port/
|
139
148
|
// incoming connections: use the remote random/unique port
|
140
149
|
// outgoing connections: use the local random/unique port
|
141
|
-
const clusterPeers = new Map<
|
150
|
+
const clusterPeers = new Map<string, Promise<RpcPeer>>();
|
151
|
+
function getClusterPeerKey(address: string, port: number) {
|
152
|
+
return `${address}:${port}`;
|
153
|
+
}
|
154
|
+
|
142
155
|
const clusterRpcServer = net.createServer(client => {
|
143
156
|
const clusterPeer = createDuplexRpcPeer(peer.selfName, 'cluster-client', client, client);
|
157
|
+
const clusterPeerAddress = client.remoteAddress;
|
144
158
|
const clusterPeerPort = client.remotePort;
|
145
|
-
|
146
|
-
|
159
|
+
const clusterPeerKey = getClusterPeerKey(clusterPeerAddress, clusterPeerPort);
|
160
|
+
// the listening peer sourceKey (client address/port) is used by the OTHER peer (the client)
|
161
|
+
// to determine if it is already connected to THIS peer (the server).
|
162
|
+
clusterPeer.onProxySerialization = (value) => onProxySerialization(value, clusterPeerKey);
|
163
|
+
clusterPeers.set(clusterPeerKey, Promise.resolve(clusterPeer));
|
147
164
|
startPluginRemoteOptions?.onClusterPeer?.(clusterPeer);
|
148
165
|
const connectRPCObject: ConnectRPCObject = async (o) => {
|
149
166
|
const sha256 = computeClusterObjectHash(o, clusterSecret);
|
150
167
|
if (sha256 !== o.sha256)
|
151
168
|
throw new Error('secret incorrect');
|
152
|
-
return resolveObject(o.proxyId, o.
|
169
|
+
return resolveObject(o.proxyId, o.sourceKey);
|
153
170
|
}
|
154
171
|
clusterPeer.params['connectRPCObject'] = connectRPCObject;
|
155
172
|
client.on('close', () => {
|
156
|
-
clusterPeers.delete(
|
173
|
+
clusterPeers.delete(clusterPeerKey);
|
157
174
|
clusterPeer.kill('cluster socket closed');
|
158
175
|
});
|
159
176
|
})
|
160
|
-
const clusterPort = await listenZero(clusterRpcServer, '127.0.0.1');
|
161
177
|
|
162
|
-
const
|
163
|
-
|
164
|
-
|
165
|
-
|
166
|
-
|
167
|
-
|
178
|
+
const listenAddress = SCRYPTED_CLUSTER_ADDRESS
|
179
|
+
? '0.0.0.0'
|
180
|
+
: '127.0.0.1';
|
181
|
+
const clusterPort = await listenZero(clusterRpcServer, listenAddress);
|
182
|
+
|
183
|
+
const ensureClusterPeer = (address: string, connectPort: number) => {
|
184
|
+
if (!address || address === SCRYPTED_CLUSTER_ADDRESS)
|
185
|
+
address = '127.0.0.1';
|
186
|
+
|
187
|
+
const clusterPeerKey = getClusterPeerKey(address, connectPort);
|
188
|
+
let clusterPeerPromise = clusterPeers.get(clusterPeerKey);
|
189
|
+
if (clusterPeerPromise)
|
190
|
+
return clusterPeerPromise;
|
191
|
+
|
192
|
+
clusterPeerPromise = (async () => {
|
193
|
+
const socket = net.connect(connectPort, address);
|
194
|
+
socket.on('close', () => clusterPeers.delete(clusterPeerKey));
|
195
|
+
|
196
|
+
try {
|
197
|
+
await once(socket, 'connect');
|
198
|
+
|
199
|
+
// this connecting peer sourceKey (server address/port) is used by the OTHER peer (the server)
|
200
|
+
// to determine if it is already connected to THIS peer (the client).
|
201
|
+
const { address: sourceAddress, port: sourcePort } = (socket.address() as net.AddressInfo);
|
202
|
+
if (sourceAddress !== SCRYPTED_CLUSTER_ADDRESS && sourceAddress !== '127.0.0.1')
|
203
|
+
console.warn("source address mismatch", sourceAddress);
|
204
|
+
const sourcePeerKey = getClusterPeerKey(sourceAddress, sourcePort);
|
205
|
+
|
206
|
+
const clusterPeer = createDuplexRpcPeer(peer.selfName, 'cluster-server', socket, socket);
|
207
|
+
clusterPeer.onProxySerialization = (value) => onProxySerialization(value, sourcePeerKey);
|
208
|
+
return clusterPeer;
|
209
|
+
}
|
210
|
+
catch (e) {
|
211
|
+
console.error('failure ipc connect', e);
|
212
|
+
socket.destroy();
|
213
|
+
throw e;
|
214
|
+
}
|
215
|
+
})();
|
168
216
|
|
169
|
-
|
170
|
-
await once(socket, 'connect');
|
171
|
-
// the sourcePort will be added to all rpc objects created by this peer session and used by resolveObject for later
|
172
|
-
// resolution when trying to find the peer.
|
173
|
-
const sourcePort = (socket.address() as net.AddressInfo).port;
|
174
|
-
|
175
|
-
const clusterPeer = createDuplexRpcPeer(peer.selfName, 'cluster-server', socket, socket);
|
176
|
-
clusterPeer.onProxySerialization = (value) => onProxySerialization(value, sourcePort);
|
177
|
-
return clusterPeer;
|
178
|
-
}
|
179
|
-
catch (e) {
|
180
|
-
console.error('failure ipc connect', e);
|
181
|
-
socket.destroy();
|
182
|
-
throw e;
|
183
|
-
}
|
184
|
-
})();
|
185
|
-
clusterPeers.set(connectPort, clusterPeerPromise);
|
186
|
-
}
|
217
|
+
clusterPeers.set(clusterPeerKey, clusterPeerPromise);
|
187
218
|
return clusterPeerPromise;
|
188
219
|
};
|
189
220
|
|
@@ -191,16 +222,16 @@ export function startPluginRemote(mainFilename: string, pluginId: string, peerSe
|
|
191
222
|
const clusterObject: ClusterObject = value?.__cluster;
|
192
223
|
if (clusterObject?.id !== clusterId)
|
193
224
|
return value;
|
194
|
-
const { port, proxyId,
|
225
|
+
const { address, port, proxyId, sourceKey } = clusterObject;
|
195
226
|
// handle the case when trying to connect to an object is on this cluster node,
|
196
227
|
// returning the actual object, rather than initiating a loopback connection.
|
197
228
|
if (port === clusterPort)
|
198
|
-
return resolveObject(proxyId,
|
229
|
+
return resolveObject(proxyId, sourceKey);
|
199
230
|
|
200
231
|
try {
|
201
|
-
const clusterPeerPromise = ensureClusterPeer(port);
|
232
|
+
const clusterPeerPromise = ensureClusterPeer(address, port);
|
202
233
|
const clusterPeer = await clusterPeerPromise;
|
203
|
-
//
|
234
|
+
// may already have this proxy so check first.
|
204
235
|
const existing = clusterPeer.remoteWeakProxies[proxyId]?.deref();
|
205
236
|
if (existing)
|
206
237
|
return existing;
|
@@ -266,7 +297,7 @@ export function startPluginRemote(mainFilename: string, pluginId: string, peerSe
|
|
266
297
|
// params.window = window;
|
267
298
|
params.exports = exports;
|
268
299
|
|
269
|
-
const entry = pluginReader(
|
300
|
+
const entry = pluginReader(`${mainNodejs}.map`)
|
270
301
|
const map = entry?.toString();
|
271
302
|
|
272
303
|
// plugins may install their own sourcemap support during startup, so
|
@@ -287,11 +318,11 @@ export function startPluginRemote(mainFilename: string, pluginId: string, peerSe
|
|
287
318
|
installSourceMapSupport({
|
288
319
|
environment: 'node',
|
289
320
|
retrieveSourceMap(source) {
|
290
|
-
if (source ===
|
321
|
+
if (source === pluginMainNodeJs || source === pluginIdMainNodeJs) {
|
291
322
|
if (!map)
|
292
323
|
return null;
|
293
324
|
return {
|
294
|
-
url:
|
325
|
+
url: pluginMainNodeJs,
|
295
326
|
map,
|
296
327
|
}
|
297
328
|
}
|
@@ -314,7 +345,7 @@ export function startPluginRemote(mainFilename: string, pluginId: string, peerSe
|
|
314
345
|
await pong(time);
|
315
346
|
};
|
316
347
|
|
317
|
-
const main = pluginReader(
|
348
|
+
const main = pluginReader(mainNodejs);
|
318
349
|
const script = main.toString();
|
319
350
|
|
320
351
|
scrypted.connect = (socket, options) => {
|
@@ -323,7 +354,7 @@ export function startPluginRemote(mainFilename: string, pluginId: string, peerSe
|
|
323
354
|
|
324
355
|
const pluginRemoteAPI: PluginRemote = scrypted.pluginRemoteAPI;
|
325
356
|
|
326
|
-
scrypted.fork = () => {
|
357
|
+
scrypted.fork = (options) => {
|
327
358
|
const ntw = new NodeThreadWorker(mainFilename, pluginId, {
|
328
359
|
packageJson,
|
329
360
|
env: process.env,
|
@@ -331,6 +362,8 @@ export function startPluginRemote(mainFilename: string, pluginId: string, peerSe
|
|
331
362
|
zipFile,
|
332
363
|
unzippedPath,
|
333
364
|
zipHash,
|
365
|
+
}, {
|
366
|
+
name: options?.name,
|
334
367
|
});
|
335
368
|
|
336
369
|
const result = (async () => {
|
@@ -358,12 +391,18 @@ export function startPluginRemote(mainFilename: string, pluginId: string, peerSe
|
|
358
391
|
|
359
392
|
const remote = await setupPluginRemote(threadPeer, forkApi, pluginId, { serverVersion }, () => systemManager.getSystemState());
|
360
393
|
forks.add(remote);
|
361
|
-
ntw.
|
394
|
+
ntw.on('exit', () => {
|
362
395
|
threadPeer.kill('worker exited');
|
363
396
|
forkApi.removeListeners();
|
364
397
|
forks.delete(remote);
|
365
398
|
allMemoryStats.delete(ntw);
|
366
399
|
});
|
400
|
+
ntw.on('error', e => {
|
401
|
+
threadPeer.kill('worker error ' + e);
|
402
|
+
forkApi.removeListeners();
|
403
|
+
forks.delete(remote);
|
404
|
+
allMemoryStats.delete(ntw);
|
405
|
+
});
|
367
406
|
|
368
407
|
for (const [nativeId, dmd] of deviceManager.nativeIds.entries()) {
|
369
408
|
await remote.setNativeId(nativeId, dmd.id, dmd.storage);
|
@@ -371,6 +410,7 @@ export function startPluginRemote(mainFilename: string, pluginId: string, peerSe
|
|
371
410
|
|
372
411
|
const forkOptions = Object.assign({}, zipOptions);
|
373
412
|
forkOptions.fork = true;
|
413
|
+
forkOptions.main = options?.filename;
|
374
414
|
return remote.loadZip(packageJson, getZip, forkOptions)
|
375
415
|
})();
|
376
416
|
|
@@ -383,7 +423,7 @@ export function startPluginRemote(mainFilename: string, pluginId: string, peerSe
|
|
383
423
|
}
|
384
424
|
|
385
425
|
try {
|
386
|
-
const filename = zipOptions?.debug ?
|
426
|
+
const filename = zipOptions?.debug ? pluginMainNodeJs : pluginIdMainNodeJs;
|
387
427
|
evalLocal(peer, script, filename, params);
|
388
428
|
|
389
429
|
if (zipOptions?.fork) {
|
@@ -0,0 +1,91 @@
|
|
1
|
+
import { getDenoPath } from '@scrypted/deno';
|
2
|
+
import child_process from 'child_process';
|
3
|
+
import path from 'path';
|
4
|
+
import { RpcMessage, RpcPeer } from "../../rpc";
|
5
|
+
import { createRpcDuplexSerializer } from '../../rpc-serializer';
|
6
|
+
import { ChildProcessWorker } from "./child-process-worker";
|
7
|
+
import { RuntimeWorkerOptions } from "./runtime-worker";
|
8
|
+
|
9
|
+
export class DenoWorker extends ChildProcessWorker {
|
10
|
+
serializer: ReturnType<typeof createRpcDuplexSerializer>;
|
11
|
+
|
12
|
+
constructor(mainFilename: string, pluginId: string, options: RuntimeWorkerOptions) {
|
13
|
+
super(pluginId, options);
|
14
|
+
|
15
|
+
const { env, pluginDebug } = options;
|
16
|
+
|
17
|
+
const execArgv: string[] = [];
|
18
|
+
if (pluginDebug) {
|
19
|
+
execArgv.push(`--inspect=0.0.0.0:${pluginDebug.inspectPort}`);
|
20
|
+
}
|
21
|
+
|
22
|
+
const args = [
|
23
|
+
'--unstable-byonm', '--unstable-bare-node-builtins', '--unstable-sloppy-imports',
|
24
|
+
'run',
|
25
|
+
...execArgv,
|
26
|
+
'--allow-all',
|
27
|
+
path.join(__dirname, '../../../deno', 'deno-plugin-remote.ts'),
|
28
|
+
// TODO: send this across.
|
29
|
+
// mainFilename.replace('dist', 'src').replace('.js', '.ts'),
|
30
|
+
'child', this.pluginId
|
31
|
+
];
|
32
|
+
this.worker = child_process.spawn(getDenoPath(), args, {
|
33
|
+
// stdio: ['pipe', 'pipe', 'pipe', 'pipe', 'pipe'],
|
34
|
+
stdio: ['pipe', 'pipe', 'pipe', 'ipc'],
|
35
|
+
env: Object.assign({
|
36
|
+
SCRYPTED_MAIN_FILENAME: mainFilename,
|
37
|
+
}, process.env, env),
|
38
|
+
serialization: 'json',
|
39
|
+
// execArgv,
|
40
|
+
});
|
41
|
+
|
42
|
+
this.worker.stderr.on('data', (data) => {
|
43
|
+
console.error(`stderr: ${data}`);
|
44
|
+
});
|
45
|
+
this.worker.stdout.on('data', (data) => {
|
46
|
+
console.log(`stdout: ${data}`);
|
47
|
+
});
|
48
|
+
|
49
|
+
this.setupWorker();
|
50
|
+
}
|
51
|
+
|
52
|
+
kill(): void {
|
53
|
+
|
54
|
+
}
|
55
|
+
|
56
|
+
setupRpcPeer(peer: RpcPeer): void {
|
57
|
+
this.worker.on('message', (message, sendHandle) => {
|
58
|
+
if ((message as any).type && sendHandle) {
|
59
|
+
peer.handleMessage(message as any, {
|
60
|
+
sendHandle,
|
61
|
+
});
|
62
|
+
}
|
63
|
+
else if (sendHandle) {
|
64
|
+
this.emit('rpc', message, sendHandle);
|
65
|
+
}
|
66
|
+
else {
|
67
|
+
peer.handleMessage(message as any);
|
68
|
+
}
|
69
|
+
});
|
70
|
+
peer.transportSafeArgumentTypes.add(Buffer.name);
|
71
|
+
peer.transportSafeArgumentTypes.add(Uint8Array.name);
|
72
|
+
}
|
73
|
+
|
74
|
+
send(message: RpcMessage, reject?: (e: Error) => void, serializationContext?: any): void {
|
75
|
+
try {
|
76
|
+
if (!this.worker)
|
77
|
+
throw new Error('fork worker has been killed');
|
78
|
+
this.worker.send(message, serializationContext?.sendHandle, e => {
|
79
|
+
if (e && reject)
|
80
|
+
reject(e);
|
81
|
+
});
|
82
|
+
}
|
83
|
+
catch (e) {
|
84
|
+
reject?.(e);
|
85
|
+
}
|
86
|
+
}
|
87
|
+
|
88
|
+
get pid() {
|
89
|
+
return this.worker?.pid;
|
90
|
+
}
|
91
|
+
}
|
@@ -1,11 +1,9 @@
|
|
1
|
-
import { RuntimeWorkerOptions as RuntimeWorkerOptions } from "./runtime-worker";
|
2
1
|
import child_process from 'child_process';
|
3
|
-
import
|
2
|
+
import net from "net";
|
4
3
|
import { RpcMessage, RpcPeer } from "../../rpc";
|
5
|
-
import { ChildProcessWorker } from "./child-process-worker";
|
6
|
-
import { getPluginNodePath } from "../plugin-npm-dependencies";
|
7
4
|
import { SidebandSocketSerializer } from "../socket-serializer";
|
8
|
-
import
|
5
|
+
import { ChildProcessWorker } from "./child-process-worker";
|
6
|
+
import { RuntimeWorkerOptions } from "./runtime-worker";
|
9
7
|
|
10
8
|
export class NodeForkWorker extends ChildProcessWorker {
|
11
9
|
|
@@ -1,24 +1,30 @@
|
|
1
1
|
import v8 from 'v8';
|
2
2
|
import worker_threads from "worker_threads";
|
3
|
-
import { EventEmitter } from "
|
3
|
+
import { EventEmitter } from "events";
|
4
4
|
import { RpcMessage, RpcPeer } from "../../rpc";
|
5
5
|
import { RuntimeWorker, RuntimeWorkerOptions } from "./runtime-worker";
|
6
6
|
|
7
7
|
export class NodeThreadWorker extends EventEmitter implements RuntimeWorker {
|
8
|
-
terminated: boolean;
|
9
8
|
worker: worker_threads.Worker;
|
9
|
+
port: worker_threads.MessagePort;
|
10
10
|
|
11
|
-
constructor(mainFilename: string, public pluginId: string, options: RuntimeWorkerOptions) {
|
11
|
+
constructor(mainFilename: string, public pluginId: string, options: RuntimeWorkerOptions, workerOptions?: worker_threads.WorkerOptions) {
|
12
12
|
super();
|
13
13
|
const { env } = options;
|
14
14
|
|
15
|
+
const message = new worker_threads.MessageChannel();
|
16
|
+
const { port1, port2 } = message;
|
15
17
|
this.worker = new worker_threads.Worker(mainFilename, {
|
16
18
|
argv: ['child-thread', this.pluginId],
|
17
19
|
env: Object.assign({}, process.env, env),
|
20
|
+
workerData: {
|
21
|
+
port: port1,
|
22
|
+
},
|
23
|
+
transferList: [port1],
|
24
|
+
...workerOptions,
|
18
25
|
});
|
19
26
|
|
20
27
|
this.worker.on('exit', () => {
|
21
|
-
this.terminated = true;
|
22
28
|
this.emit('exit');
|
23
29
|
});
|
24
30
|
this.worker.on('error', e => {
|
@@ -27,6 +33,14 @@ export class NodeThreadWorker extends EventEmitter implements RuntimeWorker {
|
|
27
33
|
this.worker.on('messageerror', e => {
|
28
34
|
this.emit('error', e);
|
29
35
|
});
|
36
|
+
|
37
|
+
this.port = port2;
|
38
|
+
this.port.on('messageerror', e => {
|
39
|
+
this.emit('error', e);
|
40
|
+
});
|
41
|
+
this.port.on('close', () => {
|
42
|
+
this.emit('error', new Error('port closed'));
|
43
|
+
});
|
30
44
|
}
|
31
45
|
|
32
46
|
get pid() {
|
@@ -48,6 +62,8 @@ export class NodeThreadWorker extends EventEmitter implements RuntimeWorker {
|
|
48
62
|
this.worker.removeAllListeners();
|
49
63
|
this.worker.stdout.removeAllListeners();
|
50
64
|
this.worker.stderr.removeAllListeners();
|
65
|
+
this.port.close();
|
66
|
+
this.port = undefined;
|
51
67
|
this.worker = undefined;
|
52
68
|
}
|
53
69
|
|
@@ -55,7 +71,7 @@ export class NodeThreadWorker extends EventEmitter implements RuntimeWorker {
|
|
55
71
|
try {
|
56
72
|
if (!this.worker)
|
57
73
|
throw new Error('thread worker has been killed');
|
58
|
-
this.
|
74
|
+
this.port.postMessage(v8.serialize(message));
|
59
75
|
}
|
60
76
|
catch (e) {
|
61
77
|
reject?.(e);
|
@@ -63,6 +79,6 @@ export class NodeThreadWorker extends EventEmitter implements RuntimeWorker {
|
|
63
79
|
}
|
64
80
|
|
65
81
|
setupRpcPeer(peer: RpcPeer): void {
|
66
|
-
this.
|
82
|
+
this.port.on('message', message => peer.handleMessage(v8.deserialize(message)));
|
67
83
|
}
|
68
84
|
}
|
package/src/rpc-peer-eval.ts
CHANGED
@@ -1,27 +1,22 @@
|
|
1
|
-
import type {
|
2
|
-
import { RpcPeer } from "./rpc";
|
1
|
+
import type { RpcPeer } from "./rpc";
|
3
2
|
|
4
|
-
|
3
|
+
export interface CompileFunctionOptions {
|
4
|
+
filename?: string;
|
5
|
+
}
|
5
6
|
|
6
7
|
function compileFunction(code: string, params?: ReadonlyArray<string>, options?: CompileFunctionOptions): any {
|
7
8
|
params = params || [];
|
8
|
-
|
9
|
+
if (options?.filename)
|
10
|
+
code = `${code}\n//# sourceURL=${options.filename}\n`;
|
11
|
+
const f = `(function(${params.join(',')}) {;${code}\n;})`;
|
9
12
|
return eval(f);
|
10
13
|
}
|
11
14
|
|
12
15
|
export function evalLocal<T>(peer: RpcPeer, script: string, filename?: string, coercedParams?: { [name: string]: any }): T {
|
13
16
|
const params = Object.assign({}, peer.params, coercedParams);
|
14
|
-
|
15
|
-
try {
|
16
|
-
// prevent bundlers from trying to include non-existent vm module.
|
17
|
-
compile = module[`require`]('vm').compileFunction;
|
18
|
-
}
|
19
|
-
catch (e) {
|
20
|
-
compile = compileFunction;
|
21
|
-
}
|
22
|
-
const f = compile(script, Object.keys(params), {
|
17
|
+
const f = compileFunction(script, Object.keys(params), {
|
23
18
|
filename,
|
24
19
|
});
|
25
20
|
const value = f(...Object.values(params));
|
26
21
|
return value;
|
27
|
-
}
|
22
|
+
}
|
package/src/rpc.ts
CHANGED
@@ -1,10 +1,10 @@
|
|
1
1
|
export function startPeriodicGarbageCollection() {
|
2
|
-
if (!
|
2
|
+
if (!globalThis.gc) {
|
3
3
|
console.warn('rpc peer garbage collection not available: global.gc is not exposed.');
|
4
4
|
}
|
5
|
-
let g: typeof
|
5
|
+
let g: typeof globalThis;
|
6
6
|
try {
|
7
|
-
g =
|
7
|
+
g = globalThis;
|
8
8
|
}
|
9
9
|
catch (e) {
|
10
10
|
}
|
package/src/runtime.ts
CHANGED
@@ -46,6 +46,7 @@ import { getNpmPackageInfo, PluginComponent } from './services/plugin';
|
|
46
46
|
import { ServiceControl } from './services/service-control';
|
47
47
|
import { UsersService } from './services/users';
|
48
48
|
import { getState, ScryptedStateManager, setState } from './state';
|
49
|
+
import { DenoWorker } from './plugin/runtime/deno-worker';
|
49
50
|
|
50
51
|
interface DeviceProxyPair {
|
51
52
|
handler: PluginDeviceProxyHandler;
|
@@ -102,6 +103,7 @@ export class ScryptedRuntime extends PluginHttp<HttpPluginData> {
|
|
102
103
|
this.pluginHosts.set('custom', (_, pluginId, options, runtime) => new CustomRuntimeWorker(pluginId, options, runtime));
|
103
104
|
this.pluginHosts.set('python', (_, pluginId, options) => new PythonRuntimeWorker(pluginId, options));
|
104
105
|
this.pluginHosts.set('node', (mainFilename, pluginId, options) => new NodeForkWorker(mainFilename, pluginId, options));
|
106
|
+
this.pluginHosts.set('deno', (mainFilename, pluginId, options) => new DenoWorker(mainFilename, pluginId, options));
|
105
107
|
|
106
108
|
app.disable('x-powered-by');
|
107
109
|
|
@@ -25,9 +25,9 @@ function start(mainFilename: string, options?: {
|
|
25
25
|
require(process.env.SCRYPTED_COMPATIBILITY_FILE);
|
26
26
|
}
|
27
27
|
|
28
|
-
if (!
|
28
|
+
if (!globalThis.gc && !process.versions.deno) {
|
29
29
|
v8.setFlagsFromString('--expose_gc')
|
30
|
-
|
30
|
+
globalThis.gc = vm.runInNewContext("gc");
|
31
31
|
}
|
32
32
|
|
33
33
|
if (!semver.gte(process.version, '16.0.0')) {
|
package/src/scrypted-main.ts
CHANGED
@@ -8,13 +8,14 @@ import { RpcMessage } from "./rpc";
|
|
8
8
|
|
9
9
|
function start(mainFilename: string) {
|
10
10
|
const pluginId = process.argv[3];
|
11
|
-
console.log('starting plugin', pluginId);
|
12
11
|
module.paths.push(getPluginNodePath(pluginId));
|
13
12
|
|
14
13
|
if (process.argv[2] === 'child-thread') {
|
15
|
-
|
14
|
+
console.log('starting thread', pluginId);
|
15
|
+
const { port } = worker_threads.workerData as { port: worker_threads.MessagePort };
|
16
|
+
const peer = startPluginRemote(mainFilename, pluginId, (message, reject) => {
|
16
17
|
try {
|
17
|
-
|
18
|
+
port.postMessage(v8.serialize(message));
|
18
19
|
}
|
19
20
|
catch (e) {
|
20
21
|
reject?.(e);
|
@@ -22,9 +23,18 @@ function start(mainFilename: string) {
|
|
22
23
|
});
|
23
24
|
peer.transportSafeArgumentTypes.add(Buffer.name);
|
24
25
|
peer.transportSafeArgumentTypes.add(Uint8Array.name);
|
25
|
-
|
26
|
+
port.on('message', message => peer.handleMessage(v8.deserialize(message)));
|
27
|
+
port.on('messageerror', e => {
|
28
|
+
console.error('message error', e);
|
29
|
+
process.exit(1);
|
30
|
+
});
|
31
|
+
port.on('close', () => {
|
32
|
+
console.error('port closed');
|
33
|
+
process.exit(1);
|
34
|
+
});
|
26
35
|
}
|
27
36
|
else {
|
37
|
+
console.log('starting plugin', pluginId);
|
28
38
|
const peer = startPluginRemote(mainFilename, process.argv[3], (message, reject, serializationContext) => process.send(message, serializationContext?.sendHandle, {
|
29
39
|
swallowErrors: !reject,
|
30
40
|
}, e => {
|