@scrypted/server 0.115.0 → 0.115.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (49) hide show
  1. package/deno/deno-plugin-remote.ts +4 -0
  2. package/dist/cluster/cluster-hash.js +1 -1
  3. package/dist/cluster/cluster-hash.js.map +1 -1
  4. package/dist/cluster/connect-rpc-object.d.ts +2 -1
  5. package/dist/plugin/plugin-api.d.ts +1 -0
  6. package/dist/plugin/plugin-remote-stats.js +19 -15
  7. package/dist/plugin/plugin-remote-stats.js.map +1 -1
  8. package/dist/plugin/plugin-remote-worker.js +90 -54
  9. package/dist/plugin/plugin-remote-worker.js.map +1 -1
  10. package/dist/plugin/runtime/deno-worker.d.ts +12 -0
  11. package/dist/plugin/runtime/deno-worker.js +84 -0
  12. package/dist/plugin/runtime/deno-worker.js.map +1 -0
  13. package/dist/plugin/runtime/node-fork-worker.d.ts +1 -1
  14. package/dist/plugin/runtime/node-fork-worker.js +2 -2
  15. package/dist/plugin/runtime/node-fork-worker.js.map +1 -1
  16. package/dist/plugin/runtime/node-thread-worker.d.ts +3 -3
  17. package/dist/plugin/runtime/node-thread-worker.js +22 -7
  18. package/dist/plugin/runtime/node-thread-worker.js.map +1 -1
  19. package/dist/rpc-peer-eval.d.ts +4 -1
  20. package/dist/rpc-peer-eval.js +4 -10
  21. package/dist/rpc-peer-eval.js.map +1 -1
  22. package/dist/rpc.d.ts +4 -1
  23. package/dist/rpc.js +8 -4
  24. package/dist/rpc.js.map +1 -1
  25. package/dist/runtime.js +2 -0
  26. package/dist/runtime.js.map +1 -1
  27. package/dist/scrypted-main-exports.js +2 -2
  28. package/dist/scrypted-main-exports.js.map +1 -1
  29. package/dist/scrypted-main.js +4 -1
  30. package/dist/scrypted-main.js.map +1 -1
  31. package/dist/scrypted-plugin-main.js +14 -4
  32. package/dist/scrypted-plugin-main.js.map +1 -1
  33. package/package.json +4 -3
  34. package/python/plugin_remote.py +423 -221
  35. package/python/rpc.py +10 -10
  36. package/src/cluster/cluster-hash.ts +1 -1
  37. package/src/cluster/connect-rpc-object.ts +2 -1
  38. package/src/plugin/plugin-api.ts +1 -0
  39. package/src/plugin/plugin-remote-stats.ts +20 -15
  40. package/src/plugin/plugin-remote-worker.ts +103 -56
  41. package/src/plugin/runtime/deno-worker.ts +91 -0
  42. package/src/plugin/runtime/node-fork-worker.ts +3 -5
  43. package/src/plugin/runtime/node-thread-worker.ts +22 -6
  44. package/src/rpc-peer-eval.ts +9 -14
  45. package/src/rpc.ts +17 -7
  46. package/src/runtime.ts +2 -0
  47. package/src/scrypted-main-exports.ts +2 -2
  48. package/src/scrypted-main.ts +4 -1
  49. package/src/scrypted-plugin-main.ts +14 -4
package/python/rpc.py CHANGED
@@ -126,7 +126,7 @@ class RpcPeer:
126
126
  self.pendingResults: Mapping[str, Future] = {}
127
127
  self.remoteWeakProxies: Mapping[str, any] = {}
128
128
  self.nameDeserializerMap: Mapping[str, RpcSerializer] = {}
129
- self.onProxySerialization: Callable[[Any, str], Any] = None
129
+ self.onProxySerialization: Callable[[Any, str], tuple[str, Any]] = None
130
130
  self.killed = False
131
131
  self.tags = {}
132
132
 
@@ -274,7 +274,7 @@ class RpcPeer:
274
274
 
275
275
  proxiedEntry = self.localProxied.get(value, None)
276
276
  if proxiedEntry:
277
- proxiedEntry['finalizerId'] = self.generateId()
277
+ proxiedEntry['finalizerId'] = RpcPeer.generateId()
278
278
  ret = {
279
279
  '__remote_proxy_id': proxiedEntry['id'],
280
280
  '__remote_proxy_finalizer_id': proxiedEntry['finalizerId'],
@@ -292,7 +292,12 @@ class RpcPeer:
292
292
  }
293
293
  return ret
294
294
 
295
- proxyId = self.generateId()
295
+ if self.onProxySerialization:
296
+ proxyId, __remote_proxy_props = self.onProxySerialization(value)
297
+ else:
298
+ __remote_proxy_props = RpcPeer.prepareProxyProperties(value)
299
+ proxyId = RpcPeer.generateId()
300
+
296
301
  proxiedEntry = {
297
302
  'id': proxyId,
298
303
  'finalizerId': proxyId,
@@ -300,11 +305,6 @@ class RpcPeer:
300
305
  self.localProxied[value] = proxiedEntry
301
306
  self.localProxyMap[proxyId] = value
302
307
 
303
- if self.onProxySerialization:
304
- __remote_proxy_props = self.onProxySerialization(value, proxyId)
305
- else:
306
- __remote_proxy_props = RpcPeer.prepareProxyProperties(value)
307
-
308
308
  ret = {
309
309
  '__remote_proxy_id': proxyId,
310
310
  '__remote_proxy_finalizer_id': proxyId,
@@ -491,7 +491,7 @@ class RpcPeer:
491
491
 
492
492
  randomDigits = string.ascii_uppercase + string.ascii_lowercase + string.digits
493
493
 
494
- def generateId(self):
494
+ def generateId():
495
495
  return ''.join(random.choices(RpcPeer.randomDigits, k=8))
496
496
 
497
497
  async def createPendingResult(self, cb: Callable[[str, Callable[[Exception], None]], None]):
@@ -500,7 +500,7 @@ class RpcPeer:
500
500
  future.set_exception(RPCResultError(None, 'RpcPeer has been killed (createPendingResult)'))
501
501
  return future
502
502
 
503
- id = self.generateId()
503
+ id = RpcPeer.generateId()
504
504
  self.pendingResults[id] = future
505
505
  await cb(id, lambda e: future.set_exception(RPCResultError(e, None)))
506
506
  return await future
@@ -2,6 +2,6 @@ import crypto from "crypto";
2
2
  import { ClusterObject } from "./connect-rpc-object";
3
3
 
4
4
  export function computeClusterObjectHash(o: ClusterObject, clusterSecret: string) {
5
- const sha256 = crypto.createHash('sha256').update(`${o.id}${o.port}${o.sourcePort || ''}${o.proxyId}${clusterSecret}`).digest().toString('base64');
5
+ const sha256 = crypto.createHash('sha256').update(`${o.id}${o.address || ''}${o.port}${o.sourceKey || ''}${o.proxyId}${clusterSecret}`).digest().toString('base64');
6
6
  return sha256;
7
7
  }
@@ -1,8 +1,9 @@
1
1
  export interface ClusterObject {
2
2
  id: string;
3
+ address: string;
3
4
  port: number;
4
5
  proxyId: string;
5
- sourcePort: number;
6
+ sourceKey: string;
6
7
  sha256: string;
7
8
  }
8
9
 
@@ -161,6 +161,7 @@ export interface PluginRemoteLoadZipOptions {
161
161
  debug?: boolean;
162
162
  zipHash: string;
163
163
  fork?: boolean;
164
+ main?: string;
164
165
 
165
166
  clusterId: string;
166
167
  clusterSecret: string;
@@ -8,23 +8,28 @@ export interface PluginStats {
8
8
 
9
9
  export function startStatsUpdater(allMemoryStats: Map<NodeThreadWorker, NodeJS.MemoryUsage>, updateStats: (stats: PluginStats) => void) {
10
10
  setInterval(() => {
11
- const cpuUsage = process.cpuUsage();
12
- allMemoryStats.set(undefined, process.memoryUsage());
11
+ let cpuUsage: NodeJS.CpuUsage;
12
+ let memoryUsage: NodeJS.MemoryUsage;
13
+ if (process.cpuUsage) {
14
+ cpuUsage = process.cpuUsage();
15
+ allMemoryStats.set(undefined, process.memoryUsage());
13
16
 
14
- const memoryUsage: NodeJS.MemoryUsage = {
15
- rss: 0,
16
- heapTotal: 0,
17
- heapUsed: 0,
18
- external: 0,
19
- arrayBuffers: 0,
20
- }
17
+ memoryUsage = {
18
+ rss: 0,
19
+ heapTotal: 0,
20
+ heapUsed: 0,
21
+ external: 0,
22
+ arrayBuffers: 0,
23
+ }
24
+
25
+ for (const mu of allMemoryStats.values()) {
26
+ memoryUsage.rss += mu.rss;
27
+ memoryUsage.heapTotal += mu.heapTotal;
28
+ memoryUsage.heapUsed += mu.heapUsed;
29
+ memoryUsage.external += mu.external;
30
+ memoryUsage.arrayBuffers += mu.arrayBuffers;
31
+ }
21
32
 
22
- for (const mu of allMemoryStats.values()) {
23
- memoryUsage.rss += mu.rss;
24
- memoryUsage.heapTotal += mu.heapTotal;
25
- memoryUsage.heapUsed += mu.heapUsed;
26
- memoryUsage.external += mu.external;
27
- memoryUsage.arrayBuffers += mu.arrayBuffers;
28
33
  }
29
34
 
30
35
  updateStats({
@@ -93,36 +93,53 @@ export function startPluginRemote(mainFilename: string, pluginId: string, peerSe
93
93
  throw new Error(`unknown service ${name}`);
94
94
  },
95
95
  async onLoadZip(scrypted: ScryptedStatic, params: any, packageJson: any, getZip: () => Promise<Buffer>, zipOptions: PluginRemoteLoadZipOptions) {
96
+ const mainFile = zipOptions?.main || 'main';
97
+ const mainNodejs = `${mainFile}.nodejs.js`;
98
+ const pluginMainNodeJs = `/plugin/${mainNodejs}`;
99
+ const pluginIdMainNodeJs = `/${pluginId}/${mainNodejs}`;
100
+
96
101
  const { clusterId, clusterSecret, zipHash } = zipOptions;
97
102
  const { zipFile, unzippedPath } = await prepareZip(getPluginVolume(pluginId), zipHash, getZip);
98
103
 
99
- const onProxySerialization = (value: any, proxyId: string, sourcePeerPort?: number) => {
104
+ const SCRYPTED_CLUSTER_ADDRESS = process.env.SCRYPTED_CLUSTER_ADDRESS;
105
+
106
+ const onProxySerialization = (value: any, sourceKey?: string) => {
100
107
  const properties = RpcPeer.prepareProxyProperties(value) || {};
101
108
  let clusterEntry: ClusterObject = properties.__cluster;
102
109
 
103
- // set the cluster identity if it does not exist.
110
+ // ensure globally stable proxyIds.
111
+ const proxyId = clusterEntry?.proxyId || RpcPeer.generateId();
112
+
113
+ // if the cluster entry already exists, check if it belongs to this node.
114
+ // if it belongs to this node, the entry must also be for this peer.
115
+ // relying on the liveness/gc of a different peer may cause race conditions.
116
+ if (clusterEntry && clusterPort === clusterEntry.port && sourceKey !== clusterEntry.sourceKey)
117
+ clusterEntry = undefined;
118
+
104
119
  if (!clusterEntry) {
105
120
  clusterEntry = {
106
121
  id: clusterId,
122
+ address: SCRYPTED_CLUSTER_ADDRESS,
107
123
  port: clusterPort,
108
124
  proxyId,
109
- sourcePort: sourcePeerPort,
125
+ sourceKey,
110
126
  sha256: null,
111
127
  };
112
128
  clusterEntry.sha256 = computeClusterObjectHash(clusterEntry, clusterSecret);
113
129
  properties.__cluster = clusterEntry;
114
130
  }
115
- // always reassign the id and source.
116
- // if this is already a p2p object, and is passed to a different peer,
117
- // a future p2p object must be routed to the correct p2p peer to find the object.
118
- // clusterEntry.proxyId = proxyId;
119
- // clusterEntry.source = source;
120
- return properties;
131
+
132
+ return {
133
+ proxyId,
134
+ properties,
135
+ };
121
136
  }
122
137
  peer.onProxySerialization = onProxySerialization;
123
138
 
124
- const resolveObject = async (id: string, sourcePeerPort: number) => {
125
- const sourcePeer = sourcePeerPort ? await clusterPeers.get(sourcePeerPort) : peer;
139
+ const resolveObject = async (id: string, sourceKey: string) => {
140
+ const sourcePeer = sourceKey
141
+ ? await clusterPeers.get(sourceKey)
142
+ : peer;
126
143
  return sourcePeer?.localProxyMap.get(id);
127
144
  }
128
145
 
@@ -130,53 +147,74 @@ export function startPluginRemote(mainFilename: string, pluginId: string, peerSe
130
147
  // on the cluster server that is listening on the actual port/
131
148
  // incoming connections: use the remote random/unique port
132
149
  // outgoing connections: use the local random/unique port
133
- const clusterPeers = new Map<number, Promise<RpcPeer>>();
150
+ const clusterPeers = new Map<string, Promise<RpcPeer>>();
151
+ function getClusterPeerKey(address: string, port: number) {
152
+ return `${address}:${port}`;
153
+ }
154
+
134
155
  const clusterRpcServer = net.createServer(client => {
135
156
  const clusterPeer = createDuplexRpcPeer(peer.selfName, 'cluster-client', client, client);
157
+ const clusterPeerAddress = client.remoteAddress;
136
158
  const clusterPeerPort = client.remotePort;
137
- clusterPeer.onProxySerialization = (value, proxyId) => onProxySerialization(value, proxyId, clusterPeerPort);
138
- clusterPeers.set(clusterPeerPort, Promise.resolve(clusterPeer));
159
+ const clusterPeerKey = getClusterPeerKey(clusterPeerAddress, clusterPeerPort);
160
+ // the listening peer sourceKey (client address/port) is used by the OTHER peer (the client)
161
+ // to determine if it is already connected to THIS peer (the server).
162
+ clusterPeer.onProxySerialization = (value) => onProxySerialization(value, clusterPeerKey);
163
+ clusterPeers.set(clusterPeerKey, Promise.resolve(clusterPeer));
139
164
  startPluginRemoteOptions?.onClusterPeer?.(clusterPeer);
140
165
  const connectRPCObject: ConnectRPCObject = async (o) => {
141
166
  const sha256 = computeClusterObjectHash(o, clusterSecret);
142
167
  if (sha256 !== o.sha256)
143
168
  throw new Error('secret incorrect');
144
- return resolveObject(o.proxyId, o.sourcePort);
169
+ return resolveObject(o.proxyId, o.sourceKey);
145
170
  }
146
171
  clusterPeer.params['connectRPCObject'] = connectRPCObject;
147
172
  client.on('close', () => {
148
- clusterPeers.delete(clusterPeerPort);
173
+ clusterPeers.delete(clusterPeerKey);
149
174
  clusterPeer.kill('cluster socket closed');
150
175
  });
151
176
  })
152
- const clusterPort = await listenZero(clusterRpcServer, '127.0.0.1');
153
177
 
154
- const ensureClusterPeer = (connectPort: number) => {
155
- let clusterPeerPromise = clusterPeers.get(connectPort);
156
- if (!clusterPeerPromise) {
157
- clusterPeerPromise = (async () => {
158
- const socket = net.connect(connectPort, '127.0.0.1');
159
- socket.on('close', () => clusterPeers.delete(connectPort));
178
+ const listenAddress = SCRYPTED_CLUSTER_ADDRESS
179
+ ? '0.0.0.0'
180
+ : '127.0.0.1';
181
+ const clusterPort = await listenZero(clusterRpcServer, listenAddress);
182
+
183
+ const ensureClusterPeer = (address: string, connectPort: number) => {
184
+ if (!address || address === SCRYPTED_CLUSTER_ADDRESS)
185
+ address = '127.0.0.1';
186
+
187
+ const clusterPeerKey = getClusterPeerKey(address, connectPort);
188
+ let clusterPeerPromise = clusterPeers.get(clusterPeerKey);
189
+ if (clusterPeerPromise)
190
+ return clusterPeerPromise;
191
+
192
+ clusterPeerPromise = (async () => {
193
+ const socket = net.connect(connectPort, address);
194
+ socket.on('close', () => clusterPeers.delete(clusterPeerKey));
195
+
196
+ try {
197
+ await once(socket, 'connect');
198
+
199
+ // this connecting peer sourceKey (server address/port) is used by the OTHER peer (the server)
200
+ // to determine if it is already connected to THIS peer (the client).
201
+ const { address: sourceAddress, port: sourcePort } = (socket.address() as net.AddressInfo);
202
+ if (sourceAddress !== SCRYPTED_CLUSTER_ADDRESS && sourceAddress !== '127.0.0.1')
203
+ console.warn("source address mismatch", sourceAddress);
204
+ const sourcePeerKey = getClusterPeerKey(sourceAddress, sourcePort);
205
+
206
+ const clusterPeer = createDuplexRpcPeer(peer.selfName, 'cluster-server', socket, socket);
207
+ clusterPeer.onProxySerialization = (value) => onProxySerialization(value, sourcePeerKey);
208
+ return clusterPeer;
209
+ }
210
+ catch (e) {
211
+ console.error('failure ipc connect', e);
212
+ socket.destroy();
213
+ throw e;
214
+ }
215
+ })();
160
216
 
161
- try {
162
- await once(socket, 'connect');
163
- // the sourcePort will be added to all rpc objects created by this peer session and used by resolveObject for later
164
- // resolution when trying to find the peer.
165
- const sourcePort = (socket.address() as net.AddressInfo).port;
166
-
167
- const clusterPeer = createDuplexRpcPeer(peer.selfName, 'cluster-server', socket, socket);
168
- clusterPeer.tags.localPort = sourcePort;
169
- clusterPeer.onProxySerialization = (value, proxyId) => onProxySerialization(value, proxyId, sourcePort);
170
- return clusterPeer;
171
- }
172
- catch (e) {
173
- console.error('failure ipc connect', e);
174
- socket.destroy();
175
- throw e;
176
- }
177
- })();
178
- clusterPeers.set(connectPort, clusterPeerPromise);
179
- }
217
+ clusterPeers.set(clusterPeerKey, clusterPeerPromise);
180
218
  return clusterPeerPromise;
181
219
  };
182
220
 
@@ -184,19 +222,19 @@ export function startPluginRemote(mainFilename: string, pluginId: string, peerSe
184
222
  const clusterObject: ClusterObject = value?.__cluster;
185
223
  if (clusterObject?.id !== clusterId)
186
224
  return value;
187
- const { port, proxyId, sourcePort } = clusterObject;
225
+ const { address, port, proxyId, sourceKey } = clusterObject;
188
226
  // handle the case when trying to connect to an object is on this cluster node,
189
227
  // returning the actual object, rather than initiating a loopback connection.
190
228
  if (port === clusterPort)
191
- return resolveObject(proxyId, sourcePort);
229
+ return resolveObject(proxyId, sourceKey);
192
230
 
193
231
  try {
194
- const clusterPeerPromise = ensureClusterPeer(port);
232
+ const clusterPeerPromise = ensureClusterPeer(address, port);
195
233
  const clusterPeer = await clusterPeerPromise;
196
- // if the localPort is the sourcePort, that means the rpc object already exists as it originated from this node.
197
- // so return the existing proxy.
198
- if (clusterPeer.tags.localPort === sourcePort)
199
- return value;
234
+ // may already have this proxy so check first.
235
+ const existing = clusterPeer.remoteWeakProxies[proxyId]?.deref();
236
+ if (existing)
237
+ return existing;
200
238
  let peerConnectRPCObject: ConnectRPCObject = clusterPeer.tags['connectRPCObject'];
201
239
  if (!peerConnectRPCObject) {
202
240
  peerConnectRPCObject = await clusterPeer.getParam('connectRPCObject');
@@ -259,7 +297,7 @@ export function startPluginRemote(mainFilename: string, pluginId: string, peerSe
259
297
  // params.window = window;
260
298
  params.exports = exports;
261
299
 
262
- const entry = pluginReader('main.nodejs.js.map')
300
+ const entry = pluginReader(`${mainNodejs}.map`)
263
301
  const map = entry?.toString();
264
302
 
265
303
  // plugins may install their own sourcemap support during startup, so
@@ -280,11 +318,11 @@ export function startPluginRemote(mainFilename: string, pluginId: string, peerSe
280
318
  installSourceMapSupport({
281
319
  environment: 'node',
282
320
  retrieveSourceMap(source) {
283
- if (source === '/plugin/main.nodejs.js' || source === `/${pluginId}/main.nodejs.js`) {
321
+ if (source === pluginMainNodeJs || source === pluginIdMainNodeJs) {
284
322
  if (!map)
285
323
  return null;
286
324
  return {
287
- url: '/plugin/main.nodejs.js',
325
+ url: pluginMainNodeJs,
288
326
  map,
289
327
  }
290
328
  }
@@ -307,7 +345,7 @@ export function startPluginRemote(mainFilename: string, pluginId: string, peerSe
307
345
  await pong(time);
308
346
  };
309
347
 
310
- const main = pluginReader('main.nodejs.js');
348
+ const main = pluginReader(mainNodejs);
311
349
  const script = main.toString();
312
350
 
313
351
  scrypted.connect = (socket, options) => {
@@ -316,7 +354,7 @@ export function startPluginRemote(mainFilename: string, pluginId: string, peerSe
316
354
 
317
355
  const pluginRemoteAPI: PluginRemote = scrypted.pluginRemoteAPI;
318
356
 
319
- scrypted.fork = () => {
357
+ scrypted.fork = (options) => {
320
358
  const ntw = new NodeThreadWorker(mainFilename, pluginId, {
321
359
  packageJson,
322
360
  env: process.env,
@@ -324,6 +362,8 @@ export function startPluginRemote(mainFilename: string, pluginId: string, peerSe
324
362
  zipFile,
325
363
  unzippedPath,
326
364
  zipHash,
365
+ }, {
366
+ name: options?.name,
327
367
  });
328
368
 
329
369
  const result = (async () => {
@@ -351,12 +391,18 @@ export function startPluginRemote(mainFilename: string, pluginId: string, peerSe
351
391
 
352
392
  const remote = await setupPluginRemote(threadPeer, forkApi, pluginId, { serverVersion }, () => systemManager.getSystemState());
353
393
  forks.add(remote);
354
- ntw.worker.on('exit', () => {
394
+ ntw.on('exit', () => {
355
395
  threadPeer.kill('worker exited');
356
396
  forkApi.removeListeners();
357
397
  forks.delete(remote);
358
398
  allMemoryStats.delete(ntw);
359
399
  });
400
+ ntw.on('error', e => {
401
+ threadPeer.kill('worker error ' + e);
402
+ forkApi.removeListeners();
403
+ forks.delete(remote);
404
+ allMemoryStats.delete(ntw);
405
+ });
360
406
 
361
407
  for (const [nativeId, dmd] of deviceManager.nativeIds.entries()) {
362
408
  await remote.setNativeId(nativeId, dmd.id, dmd.storage);
@@ -364,6 +410,7 @@ export function startPluginRemote(mainFilename: string, pluginId: string, peerSe
364
410
 
365
411
  const forkOptions = Object.assign({}, zipOptions);
366
412
  forkOptions.fork = true;
413
+ forkOptions.main = options?.filename;
367
414
  return remote.loadZip(packageJson, getZip, forkOptions)
368
415
  })();
369
416
 
@@ -376,7 +423,7 @@ export function startPluginRemote(mainFilename: string, pluginId: string, peerSe
376
423
  }
377
424
 
378
425
  try {
379
- const filename = zipOptions?.debug ? '/plugin/main.nodejs.js' : `/${pluginId}/main.nodejs.js`;
426
+ const filename = zipOptions?.debug ? pluginMainNodeJs : pluginIdMainNodeJs;
380
427
  evalLocal(peer, script, filename, params);
381
428
 
382
429
  if (zipOptions?.fork) {
@@ -0,0 +1,91 @@
1
+ import { getDenoPath } from '@scrypted/deno';
2
+ import child_process from 'child_process';
3
+ import path from 'path';
4
+ import { RpcMessage, RpcPeer } from "../../rpc";
5
+ import { createRpcDuplexSerializer } from '../../rpc-serializer';
6
+ import { ChildProcessWorker } from "./child-process-worker";
7
+ import { RuntimeWorkerOptions } from "./runtime-worker";
8
+
9
+ export class DenoWorker extends ChildProcessWorker {
10
+ serializer: ReturnType<typeof createRpcDuplexSerializer>;
11
+
12
+ constructor(mainFilename: string, pluginId: string, options: RuntimeWorkerOptions) {
13
+ super(pluginId, options);
14
+
15
+ const { env, pluginDebug } = options;
16
+
17
+ const execArgv: string[] = [];
18
+ if (pluginDebug) {
19
+ execArgv.push(`--inspect=0.0.0.0:${pluginDebug.inspectPort}`);
20
+ }
21
+
22
+ const args = [
23
+ '--unstable-byonm', '--unstable-bare-node-builtins', '--unstable-sloppy-imports',
24
+ 'run',
25
+ ...execArgv,
26
+ '--allow-all',
27
+ path.join(__dirname, '../../../deno', 'deno-plugin-remote.ts'),
28
+ // TODO: send this across.
29
+ // mainFilename.replace('dist', 'src').replace('.js', '.ts'),
30
+ 'child', this.pluginId
31
+ ];
32
+ this.worker = child_process.spawn(getDenoPath(), args, {
33
+ // stdio: ['pipe', 'pipe', 'pipe', 'pipe', 'pipe'],
34
+ stdio: ['pipe', 'pipe', 'pipe', 'ipc'],
35
+ env: Object.assign({
36
+ SCRYPTED_MAIN_FILENAME: mainFilename,
37
+ }, process.env, env),
38
+ serialization: 'json',
39
+ // execArgv,
40
+ });
41
+
42
+ this.worker.stderr.on('data', (data) => {
43
+ console.error(`stderr: ${data}`);
44
+ });
45
+ this.worker.stdout.on('data', (data) => {
46
+ console.log(`stdout: ${data}`);
47
+ });
48
+
49
+ this.setupWorker();
50
+ }
51
+
52
+ kill(): void {
53
+
54
+ }
55
+
56
+ setupRpcPeer(peer: RpcPeer): void {
57
+ this.worker.on('message', (message, sendHandle) => {
58
+ if ((message as any).type && sendHandle) {
59
+ peer.handleMessage(message as any, {
60
+ sendHandle,
61
+ });
62
+ }
63
+ else if (sendHandle) {
64
+ this.emit('rpc', message, sendHandle);
65
+ }
66
+ else {
67
+ peer.handleMessage(message as any);
68
+ }
69
+ });
70
+ peer.transportSafeArgumentTypes.add(Buffer.name);
71
+ peer.transportSafeArgumentTypes.add(Uint8Array.name);
72
+ }
73
+
74
+ send(message: RpcMessage, reject?: (e: Error) => void, serializationContext?: any): void {
75
+ try {
76
+ if (!this.worker)
77
+ throw new Error('fork worker has been killed');
78
+ this.worker.send(message, serializationContext?.sendHandle, e => {
79
+ if (e && reject)
80
+ reject(e);
81
+ });
82
+ }
83
+ catch (e) {
84
+ reject?.(e);
85
+ }
86
+ }
87
+
88
+ get pid() {
89
+ return this.worker?.pid;
90
+ }
91
+ }
@@ -1,11 +1,9 @@
1
- import { RuntimeWorkerOptions as RuntimeWorkerOptions } from "./runtime-worker";
2
1
  import child_process from 'child_process';
3
- import path from 'path';
2
+ import net from "net";
4
3
  import { RpcMessage, RpcPeer } from "../../rpc";
5
- import { ChildProcessWorker } from "./child-process-worker";
6
- import { getPluginNodePath } from "../plugin-npm-dependencies";
7
4
  import { SidebandSocketSerializer } from "../socket-serializer";
8
- import net from "net";
5
+ import { ChildProcessWorker } from "./child-process-worker";
6
+ import { RuntimeWorkerOptions } from "./runtime-worker";
9
7
 
10
8
  export class NodeForkWorker extends ChildProcessWorker {
11
9
 
@@ -1,24 +1,30 @@
1
1
  import v8 from 'v8';
2
2
  import worker_threads from "worker_threads";
3
- import { EventEmitter } from "ws";
3
+ import { EventEmitter } from "events";
4
4
  import { RpcMessage, RpcPeer } from "../../rpc";
5
5
  import { RuntimeWorker, RuntimeWorkerOptions } from "./runtime-worker";
6
6
 
7
7
  export class NodeThreadWorker extends EventEmitter implements RuntimeWorker {
8
- terminated: boolean;
9
8
  worker: worker_threads.Worker;
9
+ port: worker_threads.MessagePort;
10
10
 
11
- constructor(mainFilename: string, public pluginId: string, options: RuntimeWorkerOptions) {
11
+ constructor(mainFilename: string, public pluginId: string, options: RuntimeWorkerOptions, workerOptions?: worker_threads.WorkerOptions) {
12
12
  super();
13
13
  const { env } = options;
14
14
 
15
+ const message = new worker_threads.MessageChannel();
16
+ const { port1, port2 } = message;
15
17
  this.worker = new worker_threads.Worker(mainFilename, {
16
18
  argv: ['child-thread', this.pluginId],
17
19
  env: Object.assign({}, process.env, env),
20
+ workerData: {
21
+ port: port1,
22
+ },
23
+ transferList: [port1],
24
+ ...workerOptions,
18
25
  });
19
26
 
20
27
  this.worker.on('exit', () => {
21
- this.terminated = true;
22
28
  this.emit('exit');
23
29
  });
24
30
  this.worker.on('error', e => {
@@ -27,6 +33,14 @@ export class NodeThreadWorker extends EventEmitter implements RuntimeWorker {
27
33
  this.worker.on('messageerror', e => {
28
34
  this.emit('error', e);
29
35
  });
36
+
37
+ this.port = port2;
38
+ this.port.on('messageerror', e => {
39
+ this.emit('error', e);
40
+ });
41
+ this.port.on('close', () => {
42
+ this.emit('error', new Error('port closed'));
43
+ });
30
44
  }
31
45
 
32
46
  get pid() {
@@ -48,6 +62,8 @@ export class NodeThreadWorker extends EventEmitter implements RuntimeWorker {
48
62
  this.worker.removeAllListeners();
49
63
  this.worker.stdout.removeAllListeners();
50
64
  this.worker.stderr.removeAllListeners();
65
+ this.port.close();
66
+ this.port = undefined;
51
67
  this.worker = undefined;
52
68
  }
53
69
 
@@ -55,7 +71,7 @@ export class NodeThreadWorker extends EventEmitter implements RuntimeWorker {
55
71
  try {
56
72
  if (!this.worker)
57
73
  throw new Error('thread worker has been killed');
58
- this.worker.postMessage(v8.serialize(message));
74
+ this.port.postMessage(v8.serialize(message));
59
75
  }
60
76
  catch (e) {
61
77
  reject?.(e);
@@ -63,6 +79,6 @@ export class NodeThreadWorker extends EventEmitter implements RuntimeWorker {
63
79
  }
64
80
 
65
81
  setupRpcPeer(peer: RpcPeer): void {
66
- this.worker.on('message', message => peer.handleMessage(v8.deserialize(message)));
82
+ this.port.on('message', message => peer.handleMessage(v8.deserialize(message)));
67
83
  }
68
84
  }
@@ -1,27 +1,22 @@
1
- import type { CompileFunctionOptions } from 'vm';
2
- import { RpcPeer } from "./rpc";
1
+ import type { RpcPeer } from "./rpc";
3
2
 
4
- type CompileFunction = (code: string, params?: ReadonlyArray<string>, options?: CompileFunctionOptions) => Function;
3
+ export interface CompileFunctionOptions {
4
+ filename?: string;
5
+ }
5
6
 
6
7
  function compileFunction(code: string, params?: ReadonlyArray<string>, options?: CompileFunctionOptions): any {
7
8
  params = params || [];
8
- const f = `(function(${params.join(',')}) {;${code};})`;
9
+ if (options?.filename)
10
+ code = `${code}\n//# sourceURL=${options.filename}\n`;
11
+ const f = `(function(${params.join(',')}) {;${code}\n;})`;
9
12
  return eval(f);
10
13
  }
11
14
 
12
15
  export function evalLocal<T>(peer: RpcPeer, script: string, filename?: string, coercedParams?: { [name: string]: any }): T {
13
16
  const params = Object.assign({}, peer.params, coercedParams);
14
- let compile: CompileFunction;
15
- try {
16
- // prevent bundlers from trying to include non-existent vm module.
17
- compile = module[`require`]('vm').compileFunction;
18
- }
19
- catch (e) {
20
- compile = compileFunction;
21
- }
22
- const f = compile(script, Object.keys(params), {
17
+ const f = compileFunction(script, Object.keys(params), {
23
18
  filename,
24
19
  });
25
20
  const value = f(...Object.values(params));
26
21
  return value;
27
- }
22
+ }