@scrypted/server 0.115.1 → 0.115.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (47) hide show
  1. package/deno/deno-plugin-remote.ts +4 -0
  2. package/dist/cluster/cluster-hash.js +1 -1
  3. package/dist/cluster/cluster-hash.js.map +1 -1
  4. package/dist/cluster/connect-rpc-object.d.ts +2 -1
  5. package/dist/plugin/plugin-api.d.ts +1 -0
  6. package/dist/plugin/plugin-remote-stats.js +19 -15
  7. package/dist/plugin/plugin-remote-stats.js.map +1 -1
  8. package/dist/plugin/plugin-remote-worker.js +77 -45
  9. package/dist/plugin/plugin-remote-worker.js.map +1 -1
  10. package/dist/plugin/runtime/deno-worker.d.ts +12 -0
  11. package/dist/plugin/runtime/deno-worker.js +84 -0
  12. package/dist/plugin/runtime/deno-worker.js.map +1 -0
  13. package/dist/plugin/runtime/node-fork-worker.d.ts +1 -1
  14. package/dist/plugin/runtime/node-fork-worker.js +2 -2
  15. package/dist/plugin/runtime/node-fork-worker.js.map +1 -1
  16. package/dist/plugin/runtime/node-thread-worker.d.ts +3 -3
  17. package/dist/plugin/runtime/node-thread-worker.js +22 -7
  18. package/dist/plugin/runtime/node-thread-worker.js.map +1 -1
  19. package/dist/rpc-peer-eval.d.ts +4 -1
  20. package/dist/rpc-peer-eval.js +4 -10
  21. package/dist/rpc-peer-eval.js.map +1 -1
  22. package/dist/rpc.js +2 -2
  23. package/dist/rpc.js.map +1 -1
  24. package/dist/runtime.js +2 -0
  25. package/dist/runtime.js.map +1 -1
  26. package/dist/scrypted-main-exports.js +2 -2
  27. package/dist/scrypted-main-exports.js.map +1 -1
  28. package/dist/scrypted-main.js +4 -1
  29. package/dist/scrypted-main.js.map +1 -1
  30. package/dist/scrypted-plugin-main.js +14 -4
  31. package/dist/scrypted-plugin-main.js.map +1 -1
  32. package/package.json +4 -3
  33. package/python/plugin_remote.py +406 -218
  34. package/src/cluster/cluster-hash.ts +1 -1
  35. package/src/cluster/connect-rpc-object.ts +2 -1
  36. package/src/plugin/plugin-api.ts +1 -0
  37. package/src/plugin/plugin-remote-stats.ts +20 -15
  38. package/src/plugin/plugin-remote-worker.ts +87 -47
  39. package/src/plugin/runtime/deno-worker.ts +91 -0
  40. package/src/plugin/runtime/node-fork-worker.ts +3 -5
  41. package/src/plugin/runtime/node-thread-worker.ts +22 -6
  42. package/src/rpc-peer-eval.ts +9 -14
  43. package/src/rpc.ts +3 -3
  44. package/src/runtime.ts +2 -0
  45. package/src/scrypted-main-exports.ts +2 -2
  46. package/src/scrypted-main.ts +4 -1
  47. package/src/scrypted-plugin-main.ts +14 -4
@@ -2,6 +2,6 @@ import crypto from "crypto";
2
2
  import { ClusterObject } from "./connect-rpc-object";
3
3
 
4
4
  export function computeClusterObjectHash(o: ClusterObject, clusterSecret: string) {
5
- const sha256 = crypto.createHash('sha256').update(`${o.id}${o.port}${o.sourcePort || ''}${o.proxyId}${clusterSecret}`).digest().toString('base64');
5
+ const sha256 = crypto.createHash('sha256').update(`${o.id}${o.address || ''}${o.port}${o.sourceKey || ''}${o.proxyId}${clusterSecret}`).digest().toString('base64');
6
6
  return sha256;
7
7
  }
@@ -1,8 +1,9 @@
1
1
  export interface ClusterObject {
2
2
  id: string;
3
+ address: string;
3
4
  port: number;
4
5
  proxyId: string;
5
- sourcePort: number;
6
+ sourceKey: string;
6
7
  sha256: string;
7
8
  }
8
9
 
@@ -161,6 +161,7 @@ export interface PluginRemoteLoadZipOptions {
161
161
  debug?: boolean;
162
162
  zipHash: string;
163
163
  fork?: boolean;
164
+ main?: string;
164
165
 
165
166
  clusterId: string;
166
167
  clusterSecret: string;
@@ -8,23 +8,28 @@ export interface PluginStats {
8
8
 
9
9
  export function startStatsUpdater(allMemoryStats: Map<NodeThreadWorker, NodeJS.MemoryUsage>, updateStats: (stats: PluginStats) => void) {
10
10
  setInterval(() => {
11
- const cpuUsage = process.cpuUsage();
12
- allMemoryStats.set(undefined, process.memoryUsage());
11
+ let cpuUsage: NodeJS.CpuUsage;
12
+ let memoryUsage: NodeJS.MemoryUsage;
13
+ if (process.cpuUsage) {
14
+ cpuUsage = process.cpuUsage();
15
+ allMemoryStats.set(undefined, process.memoryUsage());
13
16
 
14
- const memoryUsage: NodeJS.MemoryUsage = {
15
- rss: 0,
16
- heapTotal: 0,
17
- heapUsed: 0,
18
- external: 0,
19
- arrayBuffers: 0,
20
- }
17
+ memoryUsage = {
18
+ rss: 0,
19
+ heapTotal: 0,
20
+ heapUsed: 0,
21
+ external: 0,
22
+ arrayBuffers: 0,
23
+ }
24
+
25
+ for (const mu of allMemoryStats.values()) {
26
+ memoryUsage.rss += mu.rss;
27
+ memoryUsage.heapTotal += mu.heapTotal;
28
+ memoryUsage.heapUsed += mu.heapUsed;
29
+ memoryUsage.external += mu.external;
30
+ memoryUsage.arrayBuffers += mu.arrayBuffers;
31
+ }
21
32
 
22
- for (const mu of allMemoryStats.values()) {
23
- memoryUsage.rss += mu.rss;
24
- memoryUsage.heapTotal += mu.heapTotal;
25
- memoryUsage.heapUsed += mu.heapUsed;
26
- memoryUsage.external += mu.external;
27
- memoryUsage.arrayBuffers += mu.arrayBuffers;
28
33
  }
29
34
 
30
35
  updateStats({
@@ -93,10 +93,17 @@ export function startPluginRemote(mainFilename: string, pluginId: string, peerSe
93
93
  throw new Error(`unknown service ${name}`);
94
94
  },
95
95
  async onLoadZip(scrypted: ScryptedStatic, params: any, packageJson: any, getZip: () => Promise<Buffer>, zipOptions: PluginRemoteLoadZipOptions) {
96
+ const mainFile = zipOptions?.main || 'main';
97
+ const mainNodejs = `${mainFile}.nodejs.js`;
98
+ const pluginMainNodeJs = `/plugin/${mainNodejs}`;
99
+ const pluginIdMainNodeJs = `/${pluginId}/${mainNodejs}`;
100
+
96
101
  const { clusterId, clusterSecret, zipHash } = zipOptions;
97
102
  const { zipFile, unzippedPath } = await prepareZip(getPluginVolume(pluginId), zipHash, getZip);
98
103
 
99
- const onProxySerialization = (value: any, sourcePeerPort?: number) => {
104
+ const SCRYPTED_CLUSTER_ADDRESS = process.env.SCRYPTED_CLUSTER_ADDRESS;
105
+
106
+ const onProxySerialization = (value: any, sourceKey?: string) => {
100
107
  const properties = RpcPeer.prepareProxyProperties(value) || {};
101
108
  let clusterEntry: ClusterObject = properties.__cluster;
102
109
 
@@ -106,16 +113,16 @@ export function startPluginRemote(mainFilename: string, pluginId: string, peerSe
106
113
  // if the cluster entry already exists, check if it belongs to this node.
107
114
  // if it belongs to this node, the entry must also be for this peer.
108
115
  // relying on the liveness/gc of a different peer may cause race conditions.
109
- if (clusterEntry && clusterPort === clusterEntry.port && sourcePeerPort !== clusterEntry.sourcePort)
116
+ if (clusterEntry && clusterPort === clusterEntry.port && sourceKey !== clusterEntry.sourceKey)
110
117
  clusterEntry = undefined;
111
118
 
112
- // set the cluster identity if it does not exist.
113
119
  if (!clusterEntry) {
114
120
  clusterEntry = {
115
121
  id: clusterId,
122
+ address: SCRYPTED_CLUSTER_ADDRESS,
116
123
  port: clusterPort,
117
124
  proxyId,
118
- sourcePort: sourcePeerPort,
125
+ sourceKey,
119
126
  sha256: null,
120
127
  };
121
128
  clusterEntry.sha256 = computeClusterObjectHash(clusterEntry, clusterSecret);
@@ -129,8 +136,10 @@ export function startPluginRemote(mainFilename: string, pluginId: string, peerSe
129
136
  }
130
137
  peer.onProxySerialization = onProxySerialization;
131
138
 
132
- const resolveObject = async (id: string, sourcePeerPort: number) => {
133
- const sourcePeer = sourcePeerPort ? await clusterPeers.get(sourcePeerPort) : peer;
139
+ const resolveObject = async (id: string, sourceKey: string) => {
140
+ const sourcePeer = sourceKey
141
+ ? await clusterPeers.get(sourceKey)
142
+ : peer;
134
143
  return sourcePeer?.localProxyMap.get(id);
135
144
  }
136
145
 
@@ -138,52 +147,74 @@ export function startPluginRemote(mainFilename: string, pluginId: string, peerSe
138
147
  // on the cluster server that is listening on the actual port/
139
148
  // incoming connections: use the remote random/unique port
140
149
  // outgoing connections: use the local random/unique port
141
- const clusterPeers = new Map<number, Promise<RpcPeer>>();
150
+ const clusterPeers = new Map<string, Promise<RpcPeer>>();
151
+ function getClusterPeerKey(address: string, port: number) {
152
+ return `${address}:${port}`;
153
+ }
154
+
142
155
  const clusterRpcServer = net.createServer(client => {
143
156
  const clusterPeer = createDuplexRpcPeer(peer.selfName, 'cluster-client', client, client);
157
+ const clusterPeerAddress = client.remoteAddress;
144
158
  const clusterPeerPort = client.remotePort;
145
- clusterPeer.onProxySerialization = (value) => onProxySerialization(value, clusterPeerPort);
146
- clusterPeers.set(clusterPeerPort, Promise.resolve(clusterPeer));
159
+ const clusterPeerKey = getClusterPeerKey(clusterPeerAddress, clusterPeerPort);
160
+ // the listening peer sourceKey (client address/port) is used by the OTHER peer (the client)
161
+ // to determine if it is already connected to THIS peer (the server).
162
+ clusterPeer.onProxySerialization = (value) => onProxySerialization(value, clusterPeerKey);
163
+ clusterPeers.set(clusterPeerKey, Promise.resolve(clusterPeer));
147
164
  startPluginRemoteOptions?.onClusterPeer?.(clusterPeer);
148
165
  const connectRPCObject: ConnectRPCObject = async (o) => {
149
166
  const sha256 = computeClusterObjectHash(o, clusterSecret);
150
167
  if (sha256 !== o.sha256)
151
168
  throw new Error('secret incorrect');
152
- return resolveObject(o.proxyId, o.sourcePort);
169
+ return resolveObject(o.proxyId, o.sourceKey);
153
170
  }
154
171
  clusterPeer.params['connectRPCObject'] = connectRPCObject;
155
172
  client.on('close', () => {
156
- clusterPeers.delete(clusterPeerPort);
173
+ clusterPeers.delete(clusterPeerKey);
157
174
  clusterPeer.kill('cluster socket closed');
158
175
  });
159
176
  })
160
- const clusterPort = await listenZero(clusterRpcServer, '127.0.0.1');
161
177
 
162
- const ensureClusterPeer = (connectPort: number) => {
163
- let clusterPeerPromise = clusterPeers.get(connectPort);
164
- if (!clusterPeerPromise) {
165
- clusterPeerPromise = (async () => {
166
- const socket = net.connect(connectPort, '127.0.0.1');
167
- socket.on('close', () => clusterPeers.delete(connectPort));
178
+ const listenAddress = SCRYPTED_CLUSTER_ADDRESS
179
+ ? '0.0.0.0'
180
+ : '127.0.0.1';
181
+ const clusterPort = await listenZero(clusterRpcServer, listenAddress);
182
+
183
+ const ensureClusterPeer = (address: string, connectPort: number) => {
184
+ if (!address || address === SCRYPTED_CLUSTER_ADDRESS)
185
+ address = '127.0.0.1';
186
+
187
+ const clusterPeerKey = getClusterPeerKey(address, connectPort);
188
+ let clusterPeerPromise = clusterPeers.get(clusterPeerKey);
189
+ if (clusterPeerPromise)
190
+ return clusterPeerPromise;
191
+
192
+ clusterPeerPromise = (async () => {
193
+ const socket = net.connect(connectPort, address);
194
+ socket.on('close', () => clusterPeers.delete(clusterPeerKey));
195
+
196
+ try {
197
+ await once(socket, 'connect');
198
+
199
+ // this connecting peer sourceKey (server address/port) is used by the OTHER peer (the server)
200
+ // to determine if it is already connected to THIS peer (the client).
201
+ const { address: sourceAddress, port: sourcePort } = (socket.address() as net.AddressInfo);
202
+ if (sourceAddress !== SCRYPTED_CLUSTER_ADDRESS && sourceAddress !== '127.0.0.1')
203
+ console.warn("source address mismatch", sourceAddress);
204
+ const sourcePeerKey = getClusterPeerKey(sourceAddress, sourcePort);
205
+
206
+ const clusterPeer = createDuplexRpcPeer(peer.selfName, 'cluster-server', socket, socket);
207
+ clusterPeer.onProxySerialization = (value) => onProxySerialization(value, sourcePeerKey);
208
+ return clusterPeer;
209
+ }
210
+ catch (e) {
211
+ console.error('failure ipc connect', e);
212
+ socket.destroy();
213
+ throw e;
214
+ }
215
+ })();
168
216
 
169
- try {
170
- await once(socket, 'connect');
171
- // the sourcePort will be added to all rpc objects created by this peer session and used by resolveObject for later
172
- // resolution when trying to find the peer.
173
- const sourcePort = (socket.address() as net.AddressInfo).port;
174
-
175
- const clusterPeer = createDuplexRpcPeer(peer.selfName, 'cluster-server', socket, socket);
176
- clusterPeer.onProxySerialization = (value) => onProxySerialization(value, sourcePort);
177
- return clusterPeer;
178
- }
179
- catch (e) {
180
- console.error('failure ipc connect', e);
181
- socket.destroy();
182
- throw e;
183
- }
184
- })();
185
- clusterPeers.set(connectPort, clusterPeerPromise);
186
- }
217
+ clusterPeers.set(clusterPeerKey, clusterPeerPromise);
187
218
  return clusterPeerPromise;
188
219
  };
189
220
 
@@ -191,16 +222,16 @@ export function startPluginRemote(mainFilename: string, pluginId: string, peerSe
191
222
  const clusterObject: ClusterObject = value?.__cluster;
192
223
  if (clusterObject?.id !== clusterId)
193
224
  return value;
194
- const { port, proxyId, sourcePort } = clusterObject;
225
+ const { address, port, proxyId, sourceKey } = clusterObject;
195
226
  // handle the case when trying to connect to an object is on this cluster node,
196
227
  // returning the actual object, rather than initiating a loopback connection.
197
228
  if (port === clusterPort)
198
- return resolveObject(proxyId, sourcePort);
229
+ return resolveObject(proxyId, sourceKey);
199
230
 
200
231
  try {
201
- const clusterPeerPromise = ensureClusterPeer(port);
232
+ const clusterPeerPromise = ensureClusterPeer(address, port);
202
233
  const clusterPeer = await clusterPeerPromise;
203
- // the proxy id is guaranteed to be unique in all peers in a cluster
234
+ // may already have this proxy so check first.
204
235
  const existing = clusterPeer.remoteWeakProxies[proxyId]?.deref();
205
236
  if (existing)
206
237
  return existing;
@@ -266,7 +297,7 @@ export function startPluginRemote(mainFilename: string, pluginId: string, peerSe
266
297
  // params.window = window;
267
298
  params.exports = exports;
268
299
 
269
- const entry = pluginReader('main.nodejs.js.map')
300
+ const entry = pluginReader(`${mainNodejs}.map`)
270
301
  const map = entry?.toString();
271
302
 
272
303
  // plugins may install their own sourcemap support during startup, so
@@ -287,11 +318,11 @@ export function startPluginRemote(mainFilename: string, pluginId: string, peerSe
287
318
  installSourceMapSupport({
288
319
  environment: 'node',
289
320
  retrieveSourceMap(source) {
290
- if (source === '/plugin/main.nodejs.js' || source === `/${pluginId}/main.nodejs.js`) {
321
+ if (source === pluginMainNodeJs || source === pluginIdMainNodeJs) {
291
322
  if (!map)
292
323
  return null;
293
324
  return {
294
- url: '/plugin/main.nodejs.js',
325
+ url: pluginMainNodeJs,
295
326
  map,
296
327
  }
297
328
  }
@@ -314,7 +345,7 @@ export function startPluginRemote(mainFilename: string, pluginId: string, peerSe
314
345
  await pong(time);
315
346
  };
316
347
 
317
- const main = pluginReader('main.nodejs.js');
348
+ const main = pluginReader(mainNodejs);
318
349
  const script = main.toString();
319
350
 
320
351
  scrypted.connect = (socket, options) => {
@@ -323,7 +354,7 @@ export function startPluginRemote(mainFilename: string, pluginId: string, peerSe
323
354
 
324
355
  const pluginRemoteAPI: PluginRemote = scrypted.pluginRemoteAPI;
325
356
 
326
- scrypted.fork = () => {
357
+ scrypted.fork = (options) => {
327
358
  const ntw = new NodeThreadWorker(mainFilename, pluginId, {
328
359
  packageJson,
329
360
  env: process.env,
@@ -331,6 +362,8 @@ export function startPluginRemote(mainFilename: string, pluginId: string, peerSe
331
362
  zipFile,
332
363
  unzippedPath,
333
364
  zipHash,
365
+ }, {
366
+ name: options?.name,
334
367
  });
335
368
 
336
369
  const result = (async () => {
@@ -358,12 +391,18 @@ export function startPluginRemote(mainFilename: string, pluginId: string, peerSe
358
391
 
359
392
  const remote = await setupPluginRemote(threadPeer, forkApi, pluginId, { serverVersion }, () => systemManager.getSystemState());
360
393
  forks.add(remote);
361
- ntw.worker.on('exit', () => {
394
+ ntw.on('exit', () => {
362
395
  threadPeer.kill('worker exited');
363
396
  forkApi.removeListeners();
364
397
  forks.delete(remote);
365
398
  allMemoryStats.delete(ntw);
366
399
  });
400
+ ntw.on('error', e => {
401
+ threadPeer.kill('worker error ' + e);
402
+ forkApi.removeListeners();
403
+ forks.delete(remote);
404
+ allMemoryStats.delete(ntw);
405
+ });
367
406
 
368
407
  for (const [nativeId, dmd] of deviceManager.nativeIds.entries()) {
369
408
  await remote.setNativeId(nativeId, dmd.id, dmd.storage);
@@ -371,6 +410,7 @@ export function startPluginRemote(mainFilename: string, pluginId: string, peerSe
371
410
 
372
411
  const forkOptions = Object.assign({}, zipOptions);
373
412
  forkOptions.fork = true;
413
+ forkOptions.main = options?.filename;
374
414
  return remote.loadZip(packageJson, getZip, forkOptions)
375
415
  })();
376
416
 
@@ -383,7 +423,7 @@ export function startPluginRemote(mainFilename: string, pluginId: string, peerSe
383
423
  }
384
424
 
385
425
  try {
386
- const filename = zipOptions?.debug ? '/plugin/main.nodejs.js' : `/${pluginId}/main.nodejs.js`;
426
+ const filename = zipOptions?.debug ? pluginMainNodeJs : pluginIdMainNodeJs;
387
427
  evalLocal(peer, script, filename, params);
388
428
 
389
429
  if (zipOptions?.fork) {
@@ -0,0 +1,91 @@
1
+ import { getDenoPath } from '@scrypted/deno';
2
+ import child_process from 'child_process';
3
+ import path from 'path';
4
+ import { RpcMessage, RpcPeer } from "../../rpc";
5
+ import { createRpcDuplexSerializer } from '../../rpc-serializer';
6
+ import { ChildProcessWorker } from "./child-process-worker";
7
+ import { RuntimeWorkerOptions } from "./runtime-worker";
8
+
9
+ export class DenoWorker extends ChildProcessWorker {
10
+ serializer: ReturnType<typeof createRpcDuplexSerializer>;
11
+
12
+ constructor(mainFilename: string, pluginId: string, options: RuntimeWorkerOptions) {
13
+ super(pluginId, options);
14
+
15
+ const { env, pluginDebug } = options;
16
+
17
+ const execArgv: string[] = [];
18
+ if (pluginDebug) {
19
+ execArgv.push(`--inspect=0.0.0.0:${pluginDebug.inspectPort}`);
20
+ }
21
+
22
+ const args = [
23
+ '--unstable-byonm', '--unstable-bare-node-builtins', '--unstable-sloppy-imports',
24
+ 'run',
25
+ ...execArgv,
26
+ '--allow-all',
27
+ path.join(__dirname, '../../../deno', 'deno-plugin-remote.ts'),
28
+ // TODO: send this across.
29
+ // mainFilename.replace('dist', 'src').replace('.js', '.ts'),
30
+ 'child', this.pluginId
31
+ ];
32
+ this.worker = child_process.spawn(getDenoPath(), args, {
33
+ // stdio: ['pipe', 'pipe', 'pipe', 'pipe', 'pipe'],
34
+ stdio: ['pipe', 'pipe', 'pipe', 'ipc'],
35
+ env: Object.assign({
36
+ SCRYPTED_MAIN_FILENAME: mainFilename,
37
+ }, process.env, env),
38
+ serialization: 'json',
39
+ // execArgv,
40
+ });
41
+
42
+ this.worker.stderr.on('data', (data) => {
43
+ console.error(`stderr: ${data}`);
44
+ });
45
+ this.worker.stdout.on('data', (data) => {
46
+ console.log(`stdout: ${data}`);
47
+ });
48
+
49
+ this.setupWorker();
50
+ }
51
+
52
+ kill(): void {
53
+
54
+ }
55
+
56
+ setupRpcPeer(peer: RpcPeer): void {
57
+ this.worker.on('message', (message, sendHandle) => {
58
+ if ((message as any).type && sendHandle) {
59
+ peer.handleMessage(message as any, {
60
+ sendHandle,
61
+ });
62
+ }
63
+ else if (sendHandle) {
64
+ this.emit('rpc', message, sendHandle);
65
+ }
66
+ else {
67
+ peer.handleMessage(message as any);
68
+ }
69
+ });
70
+ peer.transportSafeArgumentTypes.add(Buffer.name);
71
+ peer.transportSafeArgumentTypes.add(Uint8Array.name);
72
+ }
73
+
74
+ send(message: RpcMessage, reject?: (e: Error) => void, serializationContext?: any): void {
75
+ try {
76
+ if (!this.worker)
77
+ throw new Error('fork worker has been killed');
78
+ this.worker.send(message, serializationContext?.sendHandle, e => {
79
+ if (e && reject)
80
+ reject(e);
81
+ });
82
+ }
83
+ catch (e) {
84
+ reject?.(e);
85
+ }
86
+ }
87
+
88
+ get pid() {
89
+ return this.worker?.pid;
90
+ }
91
+ }
@@ -1,11 +1,9 @@
1
- import { RuntimeWorkerOptions as RuntimeWorkerOptions } from "./runtime-worker";
2
1
  import child_process from 'child_process';
3
- import path from 'path';
2
+ import net from "net";
4
3
  import { RpcMessage, RpcPeer } from "../../rpc";
5
- import { ChildProcessWorker } from "./child-process-worker";
6
- import { getPluginNodePath } from "../plugin-npm-dependencies";
7
4
  import { SidebandSocketSerializer } from "../socket-serializer";
8
- import net from "net";
5
+ import { ChildProcessWorker } from "./child-process-worker";
6
+ import { RuntimeWorkerOptions } from "./runtime-worker";
9
7
 
10
8
  export class NodeForkWorker extends ChildProcessWorker {
11
9
 
@@ -1,24 +1,30 @@
1
1
  import v8 from 'v8';
2
2
  import worker_threads from "worker_threads";
3
- import { EventEmitter } from "ws";
3
+ import { EventEmitter } from "events";
4
4
  import { RpcMessage, RpcPeer } from "../../rpc";
5
5
  import { RuntimeWorker, RuntimeWorkerOptions } from "./runtime-worker";
6
6
 
7
7
  export class NodeThreadWorker extends EventEmitter implements RuntimeWorker {
8
- terminated: boolean;
9
8
  worker: worker_threads.Worker;
9
+ port: worker_threads.MessagePort;
10
10
 
11
- constructor(mainFilename: string, public pluginId: string, options: RuntimeWorkerOptions) {
11
+ constructor(mainFilename: string, public pluginId: string, options: RuntimeWorkerOptions, workerOptions?: worker_threads.WorkerOptions) {
12
12
  super();
13
13
  const { env } = options;
14
14
 
15
+ const message = new worker_threads.MessageChannel();
16
+ const { port1, port2 } = message;
15
17
  this.worker = new worker_threads.Worker(mainFilename, {
16
18
  argv: ['child-thread', this.pluginId],
17
19
  env: Object.assign({}, process.env, env),
20
+ workerData: {
21
+ port: port1,
22
+ },
23
+ transferList: [port1],
24
+ ...workerOptions,
18
25
  });
19
26
 
20
27
  this.worker.on('exit', () => {
21
- this.terminated = true;
22
28
  this.emit('exit');
23
29
  });
24
30
  this.worker.on('error', e => {
@@ -27,6 +33,14 @@ export class NodeThreadWorker extends EventEmitter implements RuntimeWorker {
27
33
  this.worker.on('messageerror', e => {
28
34
  this.emit('error', e);
29
35
  });
36
+
37
+ this.port = port2;
38
+ this.port.on('messageerror', e => {
39
+ this.emit('error', e);
40
+ });
41
+ this.port.on('close', () => {
42
+ this.emit('error', new Error('port closed'));
43
+ });
30
44
  }
31
45
 
32
46
  get pid() {
@@ -48,6 +62,8 @@ export class NodeThreadWorker extends EventEmitter implements RuntimeWorker {
48
62
  this.worker.removeAllListeners();
49
63
  this.worker.stdout.removeAllListeners();
50
64
  this.worker.stderr.removeAllListeners();
65
+ this.port.close();
66
+ this.port = undefined;
51
67
  this.worker = undefined;
52
68
  }
53
69
 
@@ -55,7 +71,7 @@ export class NodeThreadWorker extends EventEmitter implements RuntimeWorker {
55
71
  try {
56
72
  if (!this.worker)
57
73
  throw new Error('thread worker has been killed');
58
- this.worker.postMessage(v8.serialize(message));
74
+ this.port.postMessage(v8.serialize(message));
59
75
  }
60
76
  catch (e) {
61
77
  reject?.(e);
@@ -63,6 +79,6 @@ export class NodeThreadWorker extends EventEmitter implements RuntimeWorker {
63
79
  }
64
80
 
65
81
  setupRpcPeer(peer: RpcPeer): void {
66
- this.worker.on('message', message => peer.handleMessage(v8.deserialize(message)));
82
+ this.port.on('message', message => peer.handleMessage(v8.deserialize(message)));
67
83
  }
68
84
  }
@@ -1,27 +1,22 @@
1
- import type { CompileFunctionOptions } from 'vm';
2
- import { RpcPeer } from "./rpc";
1
+ import type { RpcPeer } from "./rpc";
3
2
 
4
- type CompileFunction = (code: string, params?: ReadonlyArray<string>, options?: CompileFunctionOptions) => Function;
3
+ export interface CompileFunctionOptions {
4
+ filename?: string;
5
+ }
5
6
 
6
7
  function compileFunction(code: string, params?: ReadonlyArray<string>, options?: CompileFunctionOptions): any {
7
8
  params = params || [];
8
- const f = `(function(${params.join(',')}) {;${code};})`;
9
+ if (options?.filename)
10
+ code = `${code}\n//# sourceURL=${options.filename}\n`;
11
+ const f = `(function(${params.join(',')}) {;${code}\n;})`;
9
12
  return eval(f);
10
13
  }
11
14
 
12
15
  export function evalLocal<T>(peer: RpcPeer, script: string, filename?: string, coercedParams?: { [name: string]: any }): T {
13
16
  const params = Object.assign({}, peer.params, coercedParams);
14
- let compile: CompileFunction;
15
- try {
16
- // prevent bundlers from trying to include non-existent vm module.
17
- compile = module[`require`]('vm').compileFunction;
18
- }
19
- catch (e) {
20
- compile = compileFunction;
21
- }
22
- const f = compile(script, Object.keys(params), {
17
+ const f = compileFunction(script, Object.keys(params), {
23
18
  filename,
24
19
  });
25
20
  const value = f(...Object.values(params));
26
21
  return value;
27
- }
22
+ }
package/src/rpc.ts CHANGED
@@ -1,10 +1,10 @@
1
1
  export function startPeriodicGarbageCollection() {
2
- if (!global.gc) {
2
+ if (!globalThis.gc) {
3
3
  console.warn('rpc peer garbage collection not available: global.gc is not exposed.');
4
4
  }
5
- let g: typeof global;
5
+ let g: typeof globalThis;
6
6
  try {
7
- g = global;
7
+ g = globalThis;
8
8
  }
9
9
  catch (e) {
10
10
  }
package/src/runtime.ts CHANGED
@@ -46,6 +46,7 @@ import { getNpmPackageInfo, PluginComponent } from './services/plugin';
46
46
  import { ServiceControl } from './services/service-control';
47
47
  import { UsersService } from './services/users';
48
48
  import { getState, ScryptedStateManager, setState } from './state';
49
+ import { DenoWorker } from './plugin/runtime/deno-worker';
49
50
 
50
51
  interface DeviceProxyPair {
51
52
  handler: PluginDeviceProxyHandler;
@@ -102,6 +103,7 @@ export class ScryptedRuntime extends PluginHttp<HttpPluginData> {
102
103
  this.pluginHosts.set('custom', (_, pluginId, options, runtime) => new CustomRuntimeWorker(pluginId, options, runtime));
103
104
  this.pluginHosts.set('python', (_, pluginId, options) => new PythonRuntimeWorker(pluginId, options));
104
105
  this.pluginHosts.set('node', (mainFilename, pluginId, options) => new NodeForkWorker(mainFilename, pluginId, options));
106
+ this.pluginHosts.set('deno', (mainFilename, pluginId, options) => new DenoWorker(mainFilename, pluginId, options));
105
107
 
106
108
  app.disable('x-powered-by');
107
109
 
@@ -25,9 +25,9 @@ function start(mainFilename: string, options?: {
25
25
  require(process.env.SCRYPTED_COMPATIBILITY_FILE);
26
26
  }
27
27
 
28
- if (!global.gc) {
28
+ if (!globalThis.gc && !process.versions.deno) {
29
29
  v8.setFlagsFromString('--expose_gc')
30
- global.gc = vm.runInNewContext("gc");
30
+ globalThis.gc = vm.runInNewContext("gc");
31
31
  }
32
32
 
33
33
  if (!semver.gte(process.version, '16.0.0')) {
@@ -1,3 +1,6 @@
1
1
  import start from './scrypted-main-exports';
2
2
 
3
- start(__filename);
3
+ if (process.versions.deno)
4
+ start(process.env.SCRYPTED_MAIN_FILENAME);
5
+ else
6
+ start(__filename);
@@ -8,13 +8,14 @@ import { RpcMessage } from "./rpc";
8
8
 
9
9
  function start(mainFilename: string) {
10
10
  const pluginId = process.argv[3];
11
- console.log('starting plugin', pluginId);
12
11
  module.paths.push(getPluginNodePath(pluginId));
13
12
 
14
13
  if (process.argv[2] === 'child-thread') {
15
- const peer = startPluginRemote(mainFilename, process.argv[3], (message, reject) => {
14
+ console.log('starting thread', pluginId);
15
+ const { port } = worker_threads.workerData as { port: worker_threads.MessagePort };
16
+ const peer = startPluginRemote(mainFilename, pluginId, (message, reject) => {
16
17
  try {
17
- worker_threads.parentPort.postMessage(v8.serialize(message));
18
+ port.postMessage(v8.serialize(message));
18
19
  }
19
20
  catch (e) {
20
21
  reject?.(e);
@@ -22,9 +23,18 @@ function start(mainFilename: string) {
22
23
  });
23
24
  peer.transportSafeArgumentTypes.add(Buffer.name);
24
25
  peer.transportSafeArgumentTypes.add(Uint8Array.name);
25
- worker_threads.parentPort.on('message', message => peer.handleMessage(v8.deserialize(message)));
26
+ port.on('message', message => peer.handleMessage(v8.deserialize(message)));
27
+ port.on('messageerror', e => {
28
+ console.error('message error', e);
29
+ process.exit(1);
30
+ });
31
+ port.on('close', () => {
32
+ console.error('port closed');
33
+ process.exit(1);
34
+ });
26
35
  }
27
36
  else {
37
+ console.log('starting plugin', pluginId);
28
38
  const peer = startPluginRemote(mainFilename, process.argv[3], (message, reject, serializationContext) => process.send(message, serializationContext?.sendHandle, {
29
39
  swallowErrors: !reject,
30
40
  }, e => {