@scrypted/server 0.115.0 → 0.115.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/deno/deno-plugin-remote.ts +4 -0
- package/dist/cluster/cluster-hash.js +1 -1
- package/dist/cluster/cluster-hash.js.map +1 -1
- package/dist/cluster/connect-rpc-object.d.ts +2 -1
- package/dist/plugin/plugin-api.d.ts +1 -0
- package/dist/plugin/plugin-remote-stats.js +19 -15
- package/dist/plugin/plugin-remote-stats.js.map +1 -1
- package/dist/plugin/plugin-remote-worker.js +90 -54
- package/dist/plugin/plugin-remote-worker.js.map +1 -1
- package/dist/plugin/runtime/deno-worker.d.ts +12 -0
- package/dist/plugin/runtime/deno-worker.js +84 -0
- package/dist/plugin/runtime/deno-worker.js.map +1 -0
- package/dist/plugin/runtime/node-fork-worker.d.ts +1 -1
- package/dist/plugin/runtime/node-fork-worker.js +2 -2
- package/dist/plugin/runtime/node-fork-worker.js.map +1 -1
- package/dist/plugin/runtime/node-thread-worker.d.ts +3 -3
- package/dist/plugin/runtime/node-thread-worker.js +22 -7
- package/dist/plugin/runtime/node-thread-worker.js.map +1 -1
- package/dist/rpc-peer-eval.d.ts +4 -1
- package/dist/rpc-peer-eval.js +4 -10
- package/dist/rpc-peer-eval.js.map +1 -1
- package/dist/rpc.d.ts +4 -1
- package/dist/rpc.js +8 -4
- package/dist/rpc.js.map +1 -1
- package/dist/runtime.js +2 -0
- package/dist/runtime.js.map +1 -1
- package/dist/scrypted-main-exports.js +2 -2
- package/dist/scrypted-main-exports.js.map +1 -1
- package/dist/scrypted-main.js +4 -1
- package/dist/scrypted-main.js.map +1 -1
- package/dist/scrypted-plugin-main.js +14 -4
- package/dist/scrypted-plugin-main.js.map +1 -1
- package/package.json +4 -3
- package/python/plugin_remote.py +423 -221
- package/python/rpc.py +10 -10
- package/src/cluster/cluster-hash.ts +1 -1
- package/src/cluster/connect-rpc-object.ts +2 -1
- package/src/plugin/plugin-api.ts +1 -0
- package/src/plugin/plugin-remote-stats.ts +20 -15
- package/src/plugin/plugin-remote-worker.ts +103 -56
- package/src/plugin/runtime/deno-worker.ts +91 -0
- package/src/plugin/runtime/node-fork-worker.ts +3 -5
- package/src/plugin/runtime/node-thread-worker.ts +22 -6
- package/src/rpc-peer-eval.ts +9 -14
- package/src/rpc.ts +17 -7
- package/src/runtime.ts +2 -0
- package/src/scrypted-main-exports.ts +2 -2
- package/src/scrypted-main.ts +4 -1
- package/src/scrypted-plugin-main.ts +14 -4
package/python/rpc.py
CHANGED
@@ -126,7 +126,7 @@ class RpcPeer:
|
|
126
126
|
self.pendingResults: Mapping[str, Future] = {}
|
127
127
|
self.remoteWeakProxies: Mapping[str, any] = {}
|
128
128
|
self.nameDeserializerMap: Mapping[str, RpcSerializer] = {}
|
129
|
-
self.onProxySerialization: Callable[[Any, str], Any] = None
|
129
|
+
self.onProxySerialization: Callable[[Any, str], tuple[str, Any]] = None
|
130
130
|
self.killed = False
|
131
131
|
self.tags = {}
|
132
132
|
|
@@ -274,7 +274,7 @@ class RpcPeer:
|
|
274
274
|
|
275
275
|
proxiedEntry = self.localProxied.get(value, None)
|
276
276
|
if proxiedEntry:
|
277
|
-
proxiedEntry['finalizerId'] =
|
277
|
+
proxiedEntry['finalizerId'] = RpcPeer.generateId()
|
278
278
|
ret = {
|
279
279
|
'__remote_proxy_id': proxiedEntry['id'],
|
280
280
|
'__remote_proxy_finalizer_id': proxiedEntry['finalizerId'],
|
@@ -292,7 +292,12 @@ class RpcPeer:
|
|
292
292
|
}
|
293
293
|
return ret
|
294
294
|
|
295
|
-
|
295
|
+
if self.onProxySerialization:
|
296
|
+
proxyId, __remote_proxy_props = self.onProxySerialization(value)
|
297
|
+
else:
|
298
|
+
__remote_proxy_props = RpcPeer.prepareProxyProperties(value)
|
299
|
+
proxyId = RpcPeer.generateId()
|
300
|
+
|
296
301
|
proxiedEntry = {
|
297
302
|
'id': proxyId,
|
298
303
|
'finalizerId': proxyId,
|
@@ -300,11 +305,6 @@ class RpcPeer:
|
|
300
305
|
self.localProxied[value] = proxiedEntry
|
301
306
|
self.localProxyMap[proxyId] = value
|
302
307
|
|
303
|
-
if self.onProxySerialization:
|
304
|
-
__remote_proxy_props = self.onProxySerialization(value, proxyId)
|
305
|
-
else:
|
306
|
-
__remote_proxy_props = RpcPeer.prepareProxyProperties(value)
|
307
|
-
|
308
308
|
ret = {
|
309
309
|
'__remote_proxy_id': proxyId,
|
310
310
|
'__remote_proxy_finalizer_id': proxyId,
|
@@ -491,7 +491,7 @@ class RpcPeer:
|
|
491
491
|
|
492
492
|
randomDigits = string.ascii_uppercase + string.ascii_lowercase + string.digits
|
493
493
|
|
494
|
-
def generateId(
|
494
|
+
def generateId():
|
495
495
|
return ''.join(random.choices(RpcPeer.randomDigits, k=8))
|
496
496
|
|
497
497
|
async def createPendingResult(self, cb: Callable[[str, Callable[[Exception], None]], None]):
|
@@ -500,7 +500,7 @@ class RpcPeer:
|
|
500
500
|
future.set_exception(RPCResultError(None, 'RpcPeer has been killed (createPendingResult)'))
|
501
501
|
return future
|
502
502
|
|
503
|
-
id =
|
503
|
+
id = RpcPeer.generateId()
|
504
504
|
self.pendingResults[id] = future
|
505
505
|
await cb(id, lambda e: future.set_exception(RPCResultError(e, None)))
|
506
506
|
return await future
|
@@ -2,6 +2,6 @@ import crypto from "crypto";
|
|
2
2
|
import { ClusterObject } from "./connect-rpc-object";
|
3
3
|
|
4
4
|
export function computeClusterObjectHash(o: ClusterObject, clusterSecret: string) {
|
5
|
-
const sha256 = crypto.createHash('sha256').update(`${o.id}${o.port}${o.
|
5
|
+
const sha256 = crypto.createHash('sha256').update(`${o.id}${o.address || ''}${o.port}${o.sourceKey || ''}${o.proxyId}${clusterSecret}`).digest().toString('base64');
|
6
6
|
return sha256;
|
7
7
|
}
|
package/src/plugin/plugin-api.ts
CHANGED
@@ -8,23 +8,28 @@ export interface PluginStats {
|
|
8
8
|
|
9
9
|
export function startStatsUpdater(allMemoryStats: Map<NodeThreadWorker, NodeJS.MemoryUsage>, updateStats: (stats: PluginStats) => void) {
|
10
10
|
setInterval(() => {
|
11
|
-
|
12
|
-
|
11
|
+
let cpuUsage: NodeJS.CpuUsage;
|
12
|
+
let memoryUsage: NodeJS.MemoryUsage;
|
13
|
+
if (process.cpuUsage) {
|
14
|
+
cpuUsage = process.cpuUsage();
|
15
|
+
allMemoryStats.set(undefined, process.memoryUsage());
|
13
16
|
|
14
|
-
|
15
|
-
|
16
|
-
|
17
|
-
|
18
|
-
|
19
|
-
|
20
|
-
|
17
|
+
memoryUsage = {
|
18
|
+
rss: 0,
|
19
|
+
heapTotal: 0,
|
20
|
+
heapUsed: 0,
|
21
|
+
external: 0,
|
22
|
+
arrayBuffers: 0,
|
23
|
+
}
|
24
|
+
|
25
|
+
for (const mu of allMemoryStats.values()) {
|
26
|
+
memoryUsage.rss += mu.rss;
|
27
|
+
memoryUsage.heapTotal += mu.heapTotal;
|
28
|
+
memoryUsage.heapUsed += mu.heapUsed;
|
29
|
+
memoryUsage.external += mu.external;
|
30
|
+
memoryUsage.arrayBuffers += mu.arrayBuffers;
|
31
|
+
}
|
21
32
|
|
22
|
-
for (const mu of allMemoryStats.values()) {
|
23
|
-
memoryUsage.rss += mu.rss;
|
24
|
-
memoryUsage.heapTotal += mu.heapTotal;
|
25
|
-
memoryUsage.heapUsed += mu.heapUsed;
|
26
|
-
memoryUsage.external += mu.external;
|
27
|
-
memoryUsage.arrayBuffers += mu.arrayBuffers;
|
28
33
|
}
|
29
34
|
|
30
35
|
updateStats({
|
@@ -93,36 +93,53 @@ export function startPluginRemote(mainFilename: string, pluginId: string, peerSe
|
|
93
93
|
throw new Error(`unknown service ${name}`);
|
94
94
|
},
|
95
95
|
async onLoadZip(scrypted: ScryptedStatic, params: any, packageJson: any, getZip: () => Promise<Buffer>, zipOptions: PluginRemoteLoadZipOptions) {
|
96
|
+
const mainFile = zipOptions?.main || 'main';
|
97
|
+
const mainNodejs = `${mainFile}.nodejs.js`;
|
98
|
+
const pluginMainNodeJs = `/plugin/${mainNodejs}`;
|
99
|
+
const pluginIdMainNodeJs = `/${pluginId}/${mainNodejs}`;
|
100
|
+
|
96
101
|
const { clusterId, clusterSecret, zipHash } = zipOptions;
|
97
102
|
const { zipFile, unzippedPath } = await prepareZip(getPluginVolume(pluginId), zipHash, getZip);
|
98
103
|
|
99
|
-
const
|
104
|
+
const SCRYPTED_CLUSTER_ADDRESS = process.env.SCRYPTED_CLUSTER_ADDRESS;
|
105
|
+
|
106
|
+
const onProxySerialization = (value: any, sourceKey?: string) => {
|
100
107
|
const properties = RpcPeer.prepareProxyProperties(value) || {};
|
101
108
|
let clusterEntry: ClusterObject = properties.__cluster;
|
102
109
|
|
103
|
-
//
|
110
|
+
// ensure globally stable proxyIds.
|
111
|
+
const proxyId = clusterEntry?.proxyId || RpcPeer.generateId();
|
112
|
+
|
113
|
+
// if the cluster entry already exists, check if it belongs to this node.
|
114
|
+
// if it belongs to this node, the entry must also be for this peer.
|
115
|
+
// relying on the liveness/gc of a different peer may cause race conditions.
|
116
|
+
if (clusterEntry && clusterPort === clusterEntry.port && sourceKey !== clusterEntry.sourceKey)
|
117
|
+
clusterEntry = undefined;
|
118
|
+
|
104
119
|
if (!clusterEntry) {
|
105
120
|
clusterEntry = {
|
106
121
|
id: clusterId,
|
122
|
+
address: SCRYPTED_CLUSTER_ADDRESS,
|
107
123
|
port: clusterPort,
|
108
124
|
proxyId,
|
109
|
-
|
125
|
+
sourceKey,
|
110
126
|
sha256: null,
|
111
127
|
};
|
112
128
|
clusterEntry.sha256 = computeClusterObjectHash(clusterEntry, clusterSecret);
|
113
129
|
properties.__cluster = clusterEntry;
|
114
130
|
}
|
115
|
-
|
116
|
-
|
117
|
-
|
118
|
-
|
119
|
-
|
120
|
-
return properties;
|
131
|
+
|
132
|
+
return {
|
133
|
+
proxyId,
|
134
|
+
properties,
|
135
|
+
};
|
121
136
|
}
|
122
137
|
peer.onProxySerialization = onProxySerialization;
|
123
138
|
|
124
|
-
const resolveObject = async (id: string,
|
125
|
-
const sourcePeer =
|
139
|
+
const resolveObject = async (id: string, sourceKey: string) => {
|
140
|
+
const sourcePeer = sourceKey
|
141
|
+
? await clusterPeers.get(sourceKey)
|
142
|
+
: peer;
|
126
143
|
return sourcePeer?.localProxyMap.get(id);
|
127
144
|
}
|
128
145
|
|
@@ -130,53 +147,74 @@ export function startPluginRemote(mainFilename: string, pluginId: string, peerSe
|
|
130
147
|
// on the cluster server that is listening on the actual port/
|
131
148
|
// incoming connections: use the remote random/unique port
|
132
149
|
// outgoing connections: use the local random/unique port
|
133
|
-
const clusterPeers = new Map<
|
150
|
+
const clusterPeers = new Map<string, Promise<RpcPeer>>();
|
151
|
+
function getClusterPeerKey(address: string, port: number) {
|
152
|
+
return `${address}:${port}`;
|
153
|
+
}
|
154
|
+
|
134
155
|
const clusterRpcServer = net.createServer(client => {
|
135
156
|
const clusterPeer = createDuplexRpcPeer(peer.selfName, 'cluster-client', client, client);
|
157
|
+
const clusterPeerAddress = client.remoteAddress;
|
136
158
|
const clusterPeerPort = client.remotePort;
|
137
|
-
|
138
|
-
|
159
|
+
const clusterPeerKey = getClusterPeerKey(clusterPeerAddress, clusterPeerPort);
|
160
|
+
// the listening peer sourceKey (client address/port) is used by the OTHER peer (the client)
|
161
|
+
// to determine if it is already connected to THIS peer (the server).
|
162
|
+
clusterPeer.onProxySerialization = (value) => onProxySerialization(value, clusterPeerKey);
|
163
|
+
clusterPeers.set(clusterPeerKey, Promise.resolve(clusterPeer));
|
139
164
|
startPluginRemoteOptions?.onClusterPeer?.(clusterPeer);
|
140
165
|
const connectRPCObject: ConnectRPCObject = async (o) => {
|
141
166
|
const sha256 = computeClusterObjectHash(o, clusterSecret);
|
142
167
|
if (sha256 !== o.sha256)
|
143
168
|
throw new Error('secret incorrect');
|
144
|
-
return resolveObject(o.proxyId, o.
|
169
|
+
return resolveObject(o.proxyId, o.sourceKey);
|
145
170
|
}
|
146
171
|
clusterPeer.params['connectRPCObject'] = connectRPCObject;
|
147
172
|
client.on('close', () => {
|
148
|
-
clusterPeers.delete(
|
173
|
+
clusterPeers.delete(clusterPeerKey);
|
149
174
|
clusterPeer.kill('cluster socket closed');
|
150
175
|
});
|
151
176
|
})
|
152
|
-
const clusterPort = await listenZero(clusterRpcServer, '127.0.0.1');
|
153
177
|
|
154
|
-
const
|
155
|
-
|
156
|
-
|
157
|
-
|
158
|
-
|
159
|
-
|
178
|
+
const listenAddress = SCRYPTED_CLUSTER_ADDRESS
|
179
|
+
? '0.0.0.0'
|
180
|
+
: '127.0.0.1';
|
181
|
+
const clusterPort = await listenZero(clusterRpcServer, listenAddress);
|
182
|
+
|
183
|
+
const ensureClusterPeer = (address: string, connectPort: number) => {
|
184
|
+
if (!address || address === SCRYPTED_CLUSTER_ADDRESS)
|
185
|
+
address = '127.0.0.1';
|
186
|
+
|
187
|
+
const clusterPeerKey = getClusterPeerKey(address, connectPort);
|
188
|
+
let clusterPeerPromise = clusterPeers.get(clusterPeerKey);
|
189
|
+
if (clusterPeerPromise)
|
190
|
+
return clusterPeerPromise;
|
191
|
+
|
192
|
+
clusterPeerPromise = (async () => {
|
193
|
+
const socket = net.connect(connectPort, address);
|
194
|
+
socket.on('close', () => clusterPeers.delete(clusterPeerKey));
|
195
|
+
|
196
|
+
try {
|
197
|
+
await once(socket, 'connect');
|
198
|
+
|
199
|
+
// this connecting peer sourceKey (server address/port) is used by the OTHER peer (the server)
|
200
|
+
// to determine if it is already connected to THIS peer (the client).
|
201
|
+
const { address: sourceAddress, port: sourcePort } = (socket.address() as net.AddressInfo);
|
202
|
+
if (sourceAddress !== SCRYPTED_CLUSTER_ADDRESS && sourceAddress !== '127.0.0.1')
|
203
|
+
console.warn("source address mismatch", sourceAddress);
|
204
|
+
const sourcePeerKey = getClusterPeerKey(sourceAddress, sourcePort);
|
205
|
+
|
206
|
+
const clusterPeer = createDuplexRpcPeer(peer.selfName, 'cluster-server', socket, socket);
|
207
|
+
clusterPeer.onProxySerialization = (value) => onProxySerialization(value, sourcePeerKey);
|
208
|
+
return clusterPeer;
|
209
|
+
}
|
210
|
+
catch (e) {
|
211
|
+
console.error('failure ipc connect', e);
|
212
|
+
socket.destroy();
|
213
|
+
throw e;
|
214
|
+
}
|
215
|
+
})();
|
160
216
|
|
161
|
-
|
162
|
-
await once(socket, 'connect');
|
163
|
-
// the sourcePort will be added to all rpc objects created by this peer session and used by resolveObject for later
|
164
|
-
// resolution when trying to find the peer.
|
165
|
-
const sourcePort = (socket.address() as net.AddressInfo).port;
|
166
|
-
|
167
|
-
const clusterPeer = createDuplexRpcPeer(peer.selfName, 'cluster-server', socket, socket);
|
168
|
-
clusterPeer.tags.localPort = sourcePort;
|
169
|
-
clusterPeer.onProxySerialization = (value, proxyId) => onProxySerialization(value, proxyId, sourcePort);
|
170
|
-
return clusterPeer;
|
171
|
-
}
|
172
|
-
catch (e) {
|
173
|
-
console.error('failure ipc connect', e);
|
174
|
-
socket.destroy();
|
175
|
-
throw e;
|
176
|
-
}
|
177
|
-
})();
|
178
|
-
clusterPeers.set(connectPort, clusterPeerPromise);
|
179
|
-
}
|
217
|
+
clusterPeers.set(clusterPeerKey, clusterPeerPromise);
|
180
218
|
return clusterPeerPromise;
|
181
219
|
};
|
182
220
|
|
@@ -184,19 +222,19 @@ export function startPluginRemote(mainFilename: string, pluginId: string, peerSe
|
|
184
222
|
const clusterObject: ClusterObject = value?.__cluster;
|
185
223
|
if (clusterObject?.id !== clusterId)
|
186
224
|
return value;
|
187
|
-
const { port, proxyId,
|
225
|
+
const { address, port, proxyId, sourceKey } = clusterObject;
|
188
226
|
// handle the case when trying to connect to an object is on this cluster node,
|
189
227
|
// returning the actual object, rather than initiating a loopback connection.
|
190
228
|
if (port === clusterPort)
|
191
|
-
return resolveObject(proxyId,
|
229
|
+
return resolveObject(proxyId, sourceKey);
|
192
230
|
|
193
231
|
try {
|
194
|
-
const clusterPeerPromise = ensureClusterPeer(port);
|
232
|
+
const clusterPeerPromise = ensureClusterPeer(address, port);
|
195
233
|
const clusterPeer = await clusterPeerPromise;
|
196
|
-
//
|
197
|
-
|
198
|
-
if (
|
199
|
-
return
|
234
|
+
// may already have this proxy so check first.
|
235
|
+
const existing = clusterPeer.remoteWeakProxies[proxyId]?.deref();
|
236
|
+
if (existing)
|
237
|
+
return existing;
|
200
238
|
let peerConnectRPCObject: ConnectRPCObject = clusterPeer.tags['connectRPCObject'];
|
201
239
|
if (!peerConnectRPCObject) {
|
202
240
|
peerConnectRPCObject = await clusterPeer.getParam('connectRPCObject');
|
@@ -259,7 +297,7 @@ export function startPluginRemote(mainFilename: string, pluginId: string, peerSe
|
|
259
297
|
// params.window = window;
|
260
298
|
params.exports = exports;
|
261
299
|
|
262
|
-
const entry = pluginReader(
|
300
|
+
const entry = pluginReader(`${mainNodejs}.map`)
|
263
301
|
const map = entry?.toString();
|
264
302
|
|
265
303
|
// plugins may install their own sourcemap support during startup, so
|
@@ -280,11 +318,11 @@ export function startPluginRemote(mainFilename: string, pluginId: string, peerSe
|
|
280
318
|
installSourceMapSupport({
|
281
319
|
environment: 'node',
|
282
320
|
retrieveSourceMap(source) {
|
283
|
-
if (source ===
|
321
|
+
if (source === pluginMainNodeJs || source === pluginIdMainNodeJs) {
|
284
322
|
if (!map)
|
285
323
|
return null;
|
286
324
|
return {
|
287
|
-
url:
|
325
|
+
url: pluginMainNodeJs,
|
288
326
|
map,
|
289
327
|
}
|
290
328
|
}
|
@@ -307,7 +345,7 @@ export function startPluginRemote(mainFilename: string, pluginId: string, peerSe
|
|
307
345
|
await pong(time);
|
308
346
|
};
|
309
347
|
|
310
|
-
const main = pluginReader(
|
348
|
+
const main = pluginReader(mainNodejs);
|
311
349
|
const script = main.toString();
|
312
350
|
|
313
351
|
scrypted.connect = (socket, options) => {
|
@@ -316,7 +354,7 @@ export function startPluginRemote(mainFilename: string, pluginId: string, peerSe
|
|
316
354
|
|
317
355
|
const pluginRemoteAPI: PluginRemote = scrypted.pluginRemoteAPI;
|
318
356
|
|
319
|
-
scrypted.fork = () => {
|
357
|
+
scrypted.fork = (options) => {
|
320
358
|
const ntw = new NodeThreadWorker(mainFilename, pluginId, {
|
321
359
|
packageJson,
|
322
360
|
env: process.env,
|
@@ -324,6 +362,8 @@ export function startPluginRemote(mainFilename: string, pluginId: string, peerSe
|
|
324
362
|
zipFile,
|
325
363
|
unzippedPath,
|
326
364
|
zipHash,
|
365
|
+
}, {
|
366
|
+
name: options?.name,
|
327
367
|
});
|
328
368
|
|
329
369
|
const result = (async () => {
|
@@ -351,12 +391,18 @@ export function startPluginRemote(mainFilename: string, pluginId: string, peerSe
|
|
351
391
|
|
352
392
|
const remote = await setupPluginRemote(threadPeer, forkApi, pluginId, { serverVersion }, () => systemManager.getSystemState());
|
353
393
|
forks.add(remote);
|
354
|
-
ntw.
|
394
|
+
ntw.on('exit', () => {
|
355
395
|
threadPeer.kill('worker exited');
|
356
396
|
forkApi.removeListeners();
|
357
397
|
forks.delete(remote);
|
358
398
|
allMemoryStats.delete(ntw);
|
359
399
|
});
|
400
|
+
ntw.on('error', e => {
|
401
|
+
threadPeer.kill('worker error ' + e);
|
402
|
+
forkApi.removeListeners();
|
403
|
+
forks.delete(remote);
|
404
|
+
allMemoryStats.delete(ntw);
|
405
|
+
});
|
360
406
|
|
361
407
|
for (const [nativeId, dmd] of deviceManager.nativeIds.entries()) {
|
362
408
|
await remote.setNativeId(nativeId, dmd.id, dmd.storage);
|
@@ -364,6 +410,7 @@ export function startPluginRemote(mainFilename: string, pluginId: string, peerSe
|
|
364
410
|
|
365
411
|
const forkOptions = Object.assign({}, zipOptions);
|
366
412
|
forkOptions.fork = true;
|
413
|
+
forkOptions.main = options?.filename;
|
367
414
|
return remote.loadZip(packageJson, getZip, forkOptions)
|
368
415
|
})();
|
369
416
|
|
@@ -376,7 +423,7 @@ export function startPluginRemote(mainFilename: string, pluginId: string, peerSe
|
|
376
423
|
}
|
377
424
|
|
378
425
|
try {
|
379
|
-
const filename = zipOptions?.debug ?
|
426
|
+
const filename = zipOptions?.debug ? pluginMainNodeJs : pluginIdMainNodeJs;
|
380
427
|
evalLocal(peer, script, filename, params);
|
381
428
|
|
382
429
|
if (zipOptions?.fork) {
|
@@ -0,0 +1,91 @@
|
|
1
|
+
import { getDenoPath } from '@scrypted/deno';
|
2
|
+
import child_process from 'child_process';
|
3
|
+
import path from 'path';
|
4
|
+
import { RpcMessage, RpcPeer } from "../../rpc";
|
5
|
+
import { createRpcDuplexSerializer } from '../../rpc-serializer';
|
6
|
+
import { ChildProcessWorker } from "./child-process-worker";
|
7
|
+
import { RuntimeWorkerOptions } from "./runtime-worker";
|
8
|
+
|
9
|
+
export class DenoWorker extends ChildProcessWorker {
|
10
|
+
serializer: ReturnType<typeof createRpcDuplexSerializer>;
|
11
|
+
|
12
|
+
constructor(mainFilename: string, pluginId: string, options: RuntimeWorkerOptions) {
|
13
|
+
super(pluginId, options);
|
14
|
+
|
15
|
+
const { env, pluginDebug } = options;
|
16
|
+
|
17
|
+
const execArgv: string[] = [];
|
18
|
+
if (pluginDebug) {
|
19
|
+
execArgv.push(`--inspect=0.0.0.0:${pluginDebug.inspectPort}`);
|
20
|
+
}
|
21
|
+
|
22
|
+
const args = [
|
23
|
+
'--unstable-byonm', '--unstable-bare-node-builtins', '--unstable-sloppy-imports',
|
24
|
+
'run',
|
25
|
+
...execArgv,
|
26
|
+
'--allow-all',
|
27
|
+
path.join(__dirname, '../../../deno', 'deno-plugin-remote.ts'),
|
28
|
+
// TODO: send this across.
|
29
|
+
// mainFilename.replace('dist', 'src').replace('.js', '.ts'),
|
30
|
+
'child', this.pluginId
|
31
|
+
];
|
32
|
+
this.worker = child_process.spawn(getDenoPath(), args, {
|
33
|
+
// stdio: ['pipe', 'pipe', 'pipe', 'pipe', 'pipe'],
|
34
|
+
stdio: ['pipe', 'pipe', 'pipe', 'ipc'],
|
35
|
+
env: Object.assign({
|
36
|
+
SCRYPTED_MAIN_FILENAME: mainFilename,
|
37
|
+
}, process.env, env),
|
38
|
+
serialization: 'json',
|
39
|
+
// execArgv,
|
40
|
+
});
|
41
|
+
|
42
|
+
this.worker.stderr.on('data', (data) => {
|
43
|
+
console.error(`stderr: ${data}`);
|
44
|
+
});
|
45
|
+
this.worker.stdout.on('data', (data) => {
|
46
|
+
console.log(`stdout: ${data}`);
|
47
|
+
});
|
48
|
+
|
49
|
+
this.setupWorker();
|
50
|
+
}
|
51
|
+
|
52
|
+
kill(): void {
|
53
|
+
|
54
|
+
}
|
55
|
+
|
56
|
+
setupRpcPeer(peer: RpcPeer): void {
|
57
|
+
this.worker.on('message', (message, sendHandle) => {
|
58
|
+
if ((message as any).type && sendHandle) {
|
59
|
+
peer.handleMessage(message as any, {
|
60
|
+
sendHandle,
|
61
|
+
});
|
62
|
+
}
|
63
|
+
else if (sendHandle) {
|
64
|
+
this.emit('rpc', message, sendHandle);
|
65
|
+
}
|
66
|
+
else {
|
67
|
+
peer.handleMessage(message as any);
|
68
|
+
}
|
69
|
+
});
|
70
|
+
peer.transportSafeArgumentTypes.add(Buffer.name);
|
71
|
+
peer.transportSafeArgumentTypes.add(Uint8Array.name);
|
72
|
+
}
|
73
|
+
|
74
|
+
send(message: RpcMessage, reject?: (e: Error) => void, serializationContext?: any): void {
|
75
|
+
try {
|
76
|
+
if (!this.worker)
|
77
|
+
throw new Error('fork worker has been killed');
|
78
|
+
this.worker.send(message, serializationContext?.sendHandle, e => {
|
79
|
+
if (e && reject)
|
80
|
+
reject(e);
|
81
|
+
});
|
82
|
+
}
|
83
|
+
catch (e) {
|
84
|
+
reject?.(e);
|
85
|
+
}
|
86
|
+
}
|
87
|
+
|
88
|
+
get pid() {
|
89
|
+
return this.worker?.pid;
|
90
|
+
}
|
91
|
+
}
|
@@ -1,11 +1,9 @@
|
|
1
|
-
import { RuntimeWorkerOptions as RuntimeWorkerOptions } from "./runtime-worker";
|
2
1
|
import child_process from 'child_process';
|
3
|
-
import
|
2
|
+
import net from "net";
|
4
3
|
import { RpcMessage, RpcPeer } from "../../rpc";
|
5
|
-
import { ChildProcessWorker } from "./child-process-worker";
|
6
|
-
import { getPluginNodePath } from "../plugin-npm-dependencies";
|
7
4
|
import { SidebandSocketSerializer } from "../socket-serializer";
|
8
|
-
import
|
5
|
+
import { ChildProcessWorker } from "./child-process-worker";
|
6
|
+
import { RuntimeWorkerOptions } from "./runtime-worker";
|
9
7
|
|
10
8
|
export class NodeForkWorker extends ChildProcessWorker {
|
11
9
|
|
@@ -1,24 +1,30 @@
|
|
1
1
|
import v8 from 'v8';
|
2
2
|
import worker_threads from "worker_threads";
|
3
|
-
import { EventEmitter } from "
|
3
|
+
import { EventEmitter } from "events";
|
4
4
|
import { RpcMessage, RpcPeer } from "../../rpc";
|
5
5
|
import { RuntimeWorker, RuntimeWorkerOptions } from "./runtime-worker";
|
6
6
|
|
7
7
|
export class NodeThreadWorker extends EventEmitter implements RuntimeWorker {
|
8
|
-
terminated: boolean;
|
9
8
|
worker: worker_threads.Worker;
|
9
|
+
port: worker_threads.MessagePort;
|
10
10
|
|
11
|
-
constructor(mainFilename: string, public pluginId: string, options: RuntimeWorkerOptions) {
|
11
|
+
constructor(mainFilename: string, public pluginId: string, options: RuntimeWorkerOptions, workerOptions?: worker_threads.WorkerOptions) {
|
12
12
|
super();
|
13
13
|
const { env } = options;
|
14
14
|
|
15
|
+
const message = new worker_threads.MessageChannel();
|
16
|
+
const { port1, port2 } = message;
|
15
17
|
this.worker = new worker_threads.Worker(mainFilename, {
|
16
18
|
argv: ['child-thread', this.pluginId],
|
17
19
|
env: Object.assign({}, process.env, env),
|
20
|
+
workerData: {
|
21
|
+
port: port1,
|
22
|
+
},
|
23
|
+
transferList: [port1],
|
24
|
+
...workerOptions,
|
18
25
|
});
|
19
26
|
|
20
27
|
this.worker.on('exit', () => {
|
21
|
-
this.terminated = true;
|
22
28
|
this.emit('exit');
|
23
29
|
});
|
24
30
|
this.worker.on('error', e => {
|
@@ -27,6 +33,14 @@ export class NodeThreadWorker extends EventEmitter implements RuntimeWorker {
|
|
27
33
|
this.worker.on('messageerror', e => {
|
28
34
|
this.emit('error', e);
|
29
35
|
});
|
36
|
+
|
37
|
+
this.port = port2;
|
38
|
+
this.port.on('messageerror', e => {
|
39
|
+
this.emit('error', e);
|
40
|
+
});
|
41
|
+
this.port.on('close', () => {
|
42
|
+
this.emit('error', new Error('port closed'));
|
43
|
+
});
|
30
44
|
}
|
31
45
|
|
32
46
|
get pid() {
|
@@ -48,6 +62,8 @@ export class NodeThreadWorker extends EventEmitter implements RuntimeWorker {
|
|
48
62
|
this.worker.removeAllListeners();
|
49
63
|
this.worker.stdout.removeAllListeners();
|
50
64
|
this.worker.stderr.removeAllListeners();
|
65
|
+
this.port.close();
|
66
|
+
this.port = undefined;
|
51
67
|
this.worker = undefined;
|
52
68
|
}
|
53
69
|
|
@@ -55,7 +71,7 @@ export class NodeThreadWorker extends EventEmitter implements RuntimeWorker {
|
|
55
71
|
try {
|
56
72
|
if (!this.worker)
|
57
73
|
throw new Error('thread worker has been killed');
|
58
|
-
this.
|
74
|
+
this.port.postMessage(v8.serialize(message));
|
59
75
|
}
|
60
76
|
catch (e) {
|
61
77
|
reject?.(e);
|
@@ -63,6 +79,6 @@ export class NodeThreadWorker extends EventEmitter implements RuntimeWorker {
|
|
63
79
|
}
|
64
80
|
|
65
81
|
setupRpcPeer(peer: RpcPeer): void {
|
66
|
-
this.
|
82
|
+
this.port.on('message', message => peer.handleMessage(v8.deserialize(message)));
|
67
83
|
}
|
68
84
|
}
|
package/src/rpc-peer-eval.ts
CHANGED
@@ -1,27 +1,22 @@
|
|
1
|
-
import type {
|
2
|
-
import { RpcPeer } from "./rpc";
|
1
|
+
import type { RpcPeer } from "./rpc";
|
3
2
|
|
4
|
-
|
3
|
+
export interface CompileFunctionOptions {
|
4
|
+
filename?: string;
|
5
|
+
}
|
5
6
|
|
6
7
|
function compileFunction(code: string, params?: ReadonlyArray<string>, options?: CompileFunctionOptions): any {
|
7
8
|
params = params || [];
|
8
|
-
|
9
|
+
if (options?.filename)
|
10
|
+
code = `${code}\n//# sourceURL=${options.filename}\n`;
|
11
|
+
const f = `(function(${params.join(',')}) {;${code}\n;})`;
|
9
12
|
return eval(f);
|
10
13
|
}
|
11
14
|
|
12
15
|
export function evalLocal<T>(peer: RpcPeer, script: string, filename?: string, coercedParams?: { [name: string]: any }): T {
|
13
16
|
const params = Object.assign({}, peer.params, coercedParams);
|
14
|
-
|
15
|
-
try {
|
16
|
-
// prevent bundlers from trying to include non-existent vm module.
|
17
|
-
compile = module[`require`]('vm').compileFunction;
|
18
|
-
}
|
19
|
-
catch (e) {
|
20
|
-
compile = compileFunction;
|
21
|
-
}
|
22
|
-
const f = compile(script, Object.keys(params), {
|
17
|
+
const f = compileFunction(script, Object.keys(params), {
|
23
18
|
filename,
|
24
19
|
});
|
25
20
|
const value = f(...Object.values(params));
|
26
21
|
return value;
|
27
|
-
}
|
22
|
+
}
|