@scrypted/server 0.0.138 → 0.0.142

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of @scrypted/server might be problematic. Click here for more details.

Files changed (66) hide show
  1. package/dist/http-interfaces.js +8 -4
  2. package/dist/http-interfaces.js.map +1 -1
  3. package/dist/plugin/media.js +29 -14
  4. package/dist/plugin/media.js.map +1 -1
  5. package/dist/plugin/plugin-console.js +13 -3
  6. package/dist/plugin/plugin-console.js.map +1 -1
  7. package/dist/plugin/plugin-device.js +1 -1
  8. package/dist/plugin/plugin-device.js.map +1 -1
  9. package/dist/plugin/plugin-host-api.js +1 -1
  10. package/dist/plugin/plugin-host-api.js.map +1 -1
  11. package/dist/plugin/plugin-host.js +36 -127
  12. package/dist/plugin/plugin-host.js.map +1 -1
  13. package/dist/plugin/plugin-remote-worker.js +5 -42
  14. package/dist/plugin/plugin-remote-worker.js.map +1 -1
  15. package/dist/plugin/plugin-remote.js +3 -3
  16. package/dist/plugin/plugin-remote.js.map +1 -1
  17. package/dist/plugin/runtime/child-process-worker.js +42 -0
  18. package/dist/plugin/runtime/child-process-worker.js.map +1 -0
  19. package/dist/plugin/runtime/node-fork-worker.js +51 -0
  20. package/dist/plugin/runtime/node-fork-worker.js.map +1 -0
  21. package/dist/plugin/runtime/node-thread-worker.js +73 -0
  22. package/dist/plugin/runtime/node-thread-worker.js.map +1 -0
  23. package/dist/plugin/runtime/python-worker.js +54 -0
  24. package/dist/plugin/runtime/python-worker.js.map +1 -0
  25. package/dist/plugin/runtime/runtime-worker.js +3 -0
  26. package/dist/plugin/runtime/runtime-worker.js.map +1 -0
  27. package/dist/plugin/system.js +3 -3
  28. package/dist/plugin/system.js.map +1 -1
  29. package/dist/rpc.js +79 -55
  30. package/dist/rpc.js.map +1 -1
  31. package/dist/runtime.js +1 -6
  32. package/dist/runtime.js.map +1 -1
  33. package/dist/scrypted-main.js +14 -7
  34. package/dist/scrypted-main.js.map +1 -1
  35. package/dist/scrypted-plugin-main.js +31 -4
  36. package/dist/scrypted-plugin-main.js.map +1 -1
  37. package/dist/scrypted-server-main.js +1 -1
  38. package/dist/scrypted-server-main.js.map +1 -1
  39. package/dist/threading.js +99 -0
  40. package/dist/threading.js.map +1 -0
  41. package/package.json +5 -3
  42. package/python/plugin-remote.py +31 -2
  43. package/python/rpc.py +6 -0
  44. package/src/http-interfaces.ts +10 -5
  45. package/src/plugin/media.ts +34 -16
  46. package/src/plugin/plugin-console.ts +15 -6
  47. package/src/plugin/plugin-device.ts +2 -2
  48. package/src/plugin/plugin-host-api.ts +2 -2
  49. package/src/plugin/plugin-host.ts +42 -146
  50. package/src/plugin/plugin-remote-websocket.ts +1 -1
  51. package/src/plugin/plugin-remote-worker.ts +7 -44
  52. package/src/plugin/plugin-remote.ts +5 -5
  53. package/src/plugin/runtime/child-process-worker.ts +49 -0
  54. package/src/plugin/runtime/node-fork-worker.ts +54 -0
  55. package/src/plugin/runtime/node-thread-worker.ts +76 -0
  56. package/src/plugin/runtime/python-worker.ts +67 -0
  57. package/src/plugin/runtime/runtime-worker.ts +28 -0
  58. package/src/plugin/system.ts +4 -4
  59. package/src/rpc.ts +92 -66
  60. package/src/runtime.ts +1 -8
  61. package/src/scrypted-main.ts +15 -7
  62. package/src/scrypted-plugin-main.ts +31 -5
  63. package/src/scrypted-server-main.ts +1 -1
  64. package/src/threading.ts +108 -0
  65. package/.vscode/launch.json +0 -62
  66. package/.vscode/settings.json +0 -8
@@ -12,24 +12,21 @@ import WebSocket from 'ws';
12
12
  import { sleep } from '../sleep';
13
13
  import { PluginHostAPI } from './plugin-host-api';
14
14
  import path from 'path';
15
- import child_process from 'child_process';
16
15
  import { PluginDebug } from './plugin-debug';
17
- import readline from 'readline';
18
- import { Readable, Writable } from 'stream';
19
16
  import { ensurePluginVolume, getScryptedVolume } from './plugin-volume';
20
- import { getPluginNodePath } from './plugin-npm-dependencies';
21
17
  import { ConsoleServer, createConsoleServer } from './plugin-console';
22
18
  import { LazyRemote } from './plugin-lazy-remote';
23
19
  import crypto from 'crypto';
24
20
  import fs from 'fs';
25
21
  import mkdirp from 'mkdirp';
26
22
  import rimraf from 'rimraf';
23
+ import { RuntimeWorker } from './runtime/runtime-worker';
24
+ import { PythonRuntimeWorker } from './runtime/python-worker';
25
+ import { NodeForkWorker } from './runtime/node-fork-worker';
26
+ import { NodeThreadWorker } from './runtime/node-thread-worker';
27
27
 
28
28
  export class PluginHost {
29
- static sharedWorker: child_process.ChildProcess;
30
- static sharedWorkerImmediateRestart = false;
31
-
32
- worker: child_process.ChildProcess;
29
+ worker: RuntimeWorker;
33
30
  peer: RpcPeer;
34
31
  pluginId: string;
35
32
  module: Promise<any>;
@@ -53,10 +50,8 @@ export class PluginHost {
53
50
  kill() {
54
51
  this.killed = true;
55
52
  this.api.removeListeners();
56
- // things might get a bit race prone, so clear out the shared worker before killing.
57
- if (this.worker === PluginHost.sharedWorker)
58
- PluginHost.sharedWorker = undefined;
59
- this.worker.kill('SIGKILL');
53
+ this.peer.kill('plugin killed');
54
+ this.worker.kill();
60
55
  this.io.close();
61
56
  for (const s of Object.values(this.ws)) {
62
57
  s.close();
@@ -66,14 +61,7 @@ export class PluginHost {
66
61
  const deviceIds = new Set<string>(Object.values(this.scrypted.pluginDevices).filter(d => d.pluginId === this.pluginId).map(d => d._id));
67
62
  this.scrypted.invalidateMixins(deviceIds);
68
63
 
69
- this.consoleServer?.then(server => {
70
- server.readServer.close();
71
- server.writeServer.close();
72
- for (const s of server.sockets) {
73
- s.destroy();
74
- }
75
- });
76
- setTimeout(() => this.peer.kill('plugin killed'), 500);
64
+ this.consoleServer.then(server => server.destroy());
77
65
  }
78
66
 
79
67
  toString() {
@@ -86,7 +74,7 @@ export class PluginHost {
86
74
  return pi._id;
87
75
  }
88
76
 
89
- constructor(scrypted: ScryptedRuntime, plugin: Plugin, public pluginDebug?: PluginDebug) {
77
+ constructor(scrypted: ScryptedRuntime, plugin: Plugin, pluginDebug?: PluginDebug) {
90
78
  this.scrypted = scrypted;
91
79
  this.pluginId = plugin._id;
92
80
  this.pluginName = plugin.packageJson?.name;
@@ -101,9 +89,8 @@ export class PluginHost {
101
89
  const pluginVolume = ensurePluginVolume(this.pluginId);
102
90
 
103
91
  this.startPluginHost(logger, {
104
- NODE_PATH: path.join(getPluginNodePath(this.pluginId), 'node_modules'),
105
92
  SCRYPTED_PLUGIN_VOLUME: pluginVolume,
106
- }, this.packageJson.scrypted.runtime);
93
+ }, pluginDebug);
107
94
 
108
95
  this.io.on('connection', async (socket) => {
109
96
  try {
@@ -235,137 +222,41 @@ export class PluginHost {
235
222
  });
236
223
  }
237
224
 
238
- startPluginHost(logger: Logger, env?: any, runtime?: string) {
225
+ startPluginHost(logger: Logger, env: any, pluginDebug: PluginDebug) {
239
226
  let connected = true;
240
227
 
241
- if (runtime === 'python') {
242
- const args: string[] = [
243
- '-u',
244
- ];
245
- if (this.pluginDebug) {
246
- args.push(
247
- '-m',
248
- 'debugpy',
249
- '--listen',
250
- `0.0.0.0:${this.pluginDebug.inspectPort}`,
251
- '--wait-for-client',
252
- )
253
- }
254
- args.push(
255
- path.join(__dirname, '../../python', 'plugin-remote.py'),
256
- )
257
-
258
- this.worker = child_process.spawn('python3', args, {
259
- // stdin, stdout, stderr, peer in, peer out
260
- stdio: ['pipe', 'pipe', 'pipe', 'pipe', 'pipe'],
261
- env: Object.assign({
262
- PYTHONPATH: path.join(process.cwd(), 'node_modules/@scrypted/types'),
263
- }, process.env, env),
264
- });
265
-
266
- const peerin = this.worker.stdio[3] as Writable;
267
- const peerout = this.worker.stdio[4] as Readable;
268
-
269
- peerin.on('error', e => connected = false);
270
- peerout.on('error', e => connected = false);
271
-
272
- this.peer = new RpcPeer('host', this.pluginId, (message, reject) => {
273
- if (connected) {
274
- peerin.write(JSON.stringify(message) + '\n', e => e && reject?.(e));
275
- }
276
- else if (reject) {
277
- reject(new Error('peer disconnected'));
278
- }
279
- });
280
-
281
- const readInterface = readline.createInterface({
282
- input: peerout,
283
- terminal: false,
284
- });
285
- readInterface.on('line', line => {
286
- this.peer.handleMessage(JSON.parse(line));
228
+ if (this.packageJson.scrypted.runtime === 'python') {
229
+ this.worker = new PythonRuntimeWorker(this.pluginId, {
230
+ env,
231
+ pluginDebug,
287
232
  });
288
233
  }
289
234
  else {
290
- const execArgv: string[] = process.execArgv.slice();
291
- if (this.pluginDebug) {
292
- execArgv.push(`--inspect=0.0.0.0:${this.pluginDebug.inspectPort}`);
293
- }
294
-
295
- const useSharedWorker = process.env.SCRYPTED_SHARED_WORKER &&
296
- this.packageJson.scrypted.sharedWorker !== false &&
297
- this.packageJson.scrypted.realfs !== true &&
298
- Object.keys(this.packageJson.optionalDependencies || {}).length === 0;
299
- if (useSharedWorker) {
300
- if (!PluginHost.sharedWorker) {
301
- const worker = child_process.fork(require.main.filename, ['child', '@scrypted/shared'], {
302
- stdio: ['pipe', 'pipe', 'pipe', 'ipc'],
303
- env: Object.assign({}, process.env, env),
304
- serialization: 'advanced',
305
- execArgv,
306
- });
307
- PluginHost.sharedWorker = worker;
308
- PluginHost.sharedWorker.setMaxListeners(100);
309
- const clearSharedWorker = () => {
310
- if (worker === PluginHost.sharedWorker)
311
- PluginHost.sharedWorker = undefined;
312
- };
313
- PluginHost.sharedWorker.on('close', () => clearSharedWorker);
314
- PluginHost.sharedWorker.on('error', () => clearSharedWorker);
315
- PluginHost.sharedWorker.on('exit', () => clearSharedWorker);
316
- PluginHost.sharedWorker.on('disconnect', () => clearSharedWorker);
317
- }
318
- PluginHost.sharedWorker.send({
319
- type: 'start',
320
- pluginId: this.pluginId,
321
- });
322
- this.worker = PluginHost.sharedWorker;
323
- this.worker.on('message', (message: any) => {
324
- if (message.pluginId === this.pluginId)
325
- this.peer.handleMessage(message.message)
326
- });
327
-
328
- this.peer = new RpcPeer('host', this.pluginId, (message, reject) => {
329
- if (connected) {
330
- this.worker.send({
331
- type: 'message',
332
- pluginId: this.pluginId,
333
- message: message,
334
- }, undefined, e => {
335
- if (e && reject)
336
- reject(e);
337
- });
338
- }
339
- else if (reject) {
340
- reject(new Error('peer disconnected'));
341
- }
235
+ if (!process.env.SCRYPTED_SHARED_WORKER || (this.packageJson.optionalDependencies && Object.keys(this.packageJson.optionalDependencies).length)) {
236
+ this.worker = new NodeForkWorker(this.pluginId, {
237
+ env,
238
+ pluginDebug,
342
239
  });
343
240
  }
344
241
  else {
345
- this.worker = child_process.fork(require.main.filename, ['child', this.pluginId], {
346
- stdio: ['pipe', 'pipe', 'pipe', 'ipc'],
347
- env: Object.assign({}, process.env, env),
348
- serialization: 'advanced',
349
- execArgv,
350
- });
351
- this.worker.on('message', message => this.peer.handleMessage(message as any));
352
-
353
- this.peer = new RpcPeer('host', this.pluginId, (message, reject) => {
354
- if (connected) {
355
- this.worker.send(message, undefined, e => {
356
- if (e && reject)
357
- reject(e);
358
- });
359
- }
360
- else if (reject) {
361
- reject(new Error('peer disconnected'));
362
- }
242
+ this.worker = new NodeThreadWorker(this.pluginId, {
243
+ env,
244
+ pluginDebug,
363
245
  });
364
246
  }
365
-
366
- this.peer.transportSafeArgumentTypes.add(Buffer.name);
367
247
  }
368
248
 
249
+ this.peer = new RpcPeer('host', this.pluginId, (message, reject) => {
250
+ if (connected) {
251
+ this.worker.send(message, reject);
252
+ }
253
+ else if (reject) {
254
+ reject(new Error('peer disconnected'));
255
+ }
256
+ });
257
+
258
+ this.worker.setupRpcPeer(this.peer);
259
+
369
260
  this.worker.stdout.on('data', data => console.log(data.toString()));
370
261
  this.worker.stderr.on('data', data => console.error(data.toString()));
371
262
  this.consoleServer = createConsoleServer(this.worker.stdout, this.worker.stderr);
@@ -375,21 +266,26 @@ export class PluginHost {
375
266
  pluginConsole.log('starting plugin', this.pluginId, this.packageJson.version);
376
267
  });
377
268
 
378
- this.worker.on('close', () => {
269
+ const disconnect = () => {
379
270
  connected = false;
271
+ this.peer.kill('plugin disconnected');
272
+ };
273
+
274
+ this.worker.on('close', () => {
380
275
  logger.log('e', `${this.pluginName} close`);
276
+ disconnect();
381
277
  });
382
278
  this.worker.on('disconnect', () => {
383
- connected = false;
384
279
  logger.log('e', `${this.pluginName} disconnected`);
280
+ disconnect();
385
281
  });
386
282
  this.worker.on('exit', async (code, signal) => {
387
- connected = false;
388
283
  logger.log('e', `${this.pluginName} exited ${code} ${signal}`);
284
+ disconnect();
389
285
  });
390
286
  this.worker.on('error', e => {
391
- connected = false;
392
287
  logger.log('e', `${this.pluginName} error ${e}`);
288
+ disconnect();
393
289
  });
394
290
 
395
291
  this.peer.onOob = (oob: any) => {
@@ -69,7 +69,7 @@ export interface WebSocketMethods {
69
69
  close(message: string): void;
70
70
  }
71
71
 
72
- export function createWebSocketClass(__websocketConnect: WebSocketConnect) {
72
+ export function createWebSocketClass(__websocketConnect: WebSocketConnect): any {
73
73
 
74
74
  // @ts-ignore
75
75
  class WebSocket extends WebSocketEventTarget {
@@ -10,50 +10,8 @@ import net from 'net'
10
10
  import { installOptionalDependencies } from './plugin-npm-dependencies';
11
11
  import { createREPLServer } from './plugin-repl';
12
12
 
13
- export function startSharedPluginRemote() {
14
- process.setMaxListeners(100);
15
- process.on('message', (message: any) => {
16
- if (message.type === 'start')
17
- startPluginRemote(message.pluginId)
18
- });
19
- }
20
-
21
- export function startPluginRemote(pluginId: string) {
22
- let peerSend: (message: RpcMessage, reject?: (e: Error) => void) => void;
23
- let peerListener: NodeJS.MessageListener;
24
-
25
- if (process.argv[3] === '@scrypted/shared') {
26
- peerSend = (message, reject) => process.send({
27
- pluginId,
28
- message,
29
- }, undefined, {
30
- swallowErrors: !reject,
31
- }, e => {
32
- if (e)
33
- reject?.(e);
34
- });
35
- peerListener = (message: any) => {
36
- if (message.type === 'message' && message.pluginId === pluginId)
37
- peer.handleMessage(message.message);
38
- }
39
- }
40
- else {
41
- peerSend = (message, reject) => process.send(message, undefined, {
42
- swallowErrors: !reject,
43
- }, e => {
44
- if (e)
45
- reject?.(e);
46
- });
47
- peerListener = message => peer.handleMessage(message as RpcMessage);
48
- }
49
-
13
+ export function startPluginRemote(pluginId: string, peerSend: (message: RpcMessage, reject?: (e: Error) => void) => void) {
50
14
  const peer = new RpcPeer('unknown', 'host', peerSend);
51
- peer.transportSafeArgumentTypes.add(Buffer.name);
52
- process.on('message', peerListener);
53
- process.on('disconnect', () => {
54
- console.error('peer host disconnected, exiting.');
55
- process.exit(1);
56
- });
57
15
 
58
16
  let systemManager: SystemManager;
59
17
  let deviceManager: DeviceManager;
@@ -259,6 +217,9 @@ export function startPluginRemote(pluginId: string) {
259
217
  systemManager = scrypted.systemManager;
260
218
  deviceManager = scrypted.deviceManager;
261
219
 
220
+ process.removeAllListeners('uncaughtException');
221
+ process.removeAllListeners('unhandledRejection');
222
+
262
223
  process.on('uncaughtException', e => {
263
224
  getPluginConsole().error('uncaughtException', e);
264
225
  scrypted.log.e('uncaughtException ' + e?.toString());
@@ -267,5 +228,7 @@ export function startPluginRemote(pluginId: string) {
267
228
  getPluginConsole().error('unhandledRejection', e);
268
229
  scrypted.log.e('unhandledRejection ' + e?.toString());
269
230
  });
270
- })
231
+ });
232
+
233
+ return peer;
271
234
  }
@@ -4,7 +4,7 @@ import path from 'path';
4
4
  import { ScryptedNativeId, DeviceManager, Logger, Device, DeviceManifest, DeviceState, EndpointManager, SystemDeviceState, ScryptedStatic, SystemManager, MediaManager, ScryptedMimeTypes, ScryptedInterface, ScryptedInterfaceProperty, HttpRequest } from '@scrypted/types'
5
5
  import { PluginAPI, PluginLogger, PluginRemote, PluginRemoteLoadZipOptions } from './plugin-api';
6
6
  import { SystemManagerImpl } from './system';
7
- import { RpcPeer, RPCResultError, PROPERTY_PROXY_ONEWAY_METHODS, PROPERTY_JSON_DISABLE_SERIALIZATION } from '../rpc';
7
+ import { RpcPeer, RPCResultError } from '../rpc';
8
8
  import { BufferSerializer } from './buffer-serializer';
9
9
  import { createWebSocketClass, WebSocketConnectCallbacks, WebSocketMethods } from './plugin-remote-websocket';
10
10
  import fs from 'fs';
@@ -88,7 +88,7 @@ class EndpointManagerImpl implements EndpointManager {
88
88
  }
89
89
  async getPublicCloudEndpoint(nativeId?: ScryptedNativeId): Promise<string> {
90
90
  const local = await this.getPublicLocalEndpoint(nativeId);
91
- const mo = this.mediaManager.createMediaObject(local, ScryptedMimeTypes.LocalUrl);
91
+ const mo = this.mediaManager.createMediaObject(Buffer.from(local), ScryptedMimeTypes.LocalUrl);
92
92
  return this.mediaManager.convertMediaObjectToUrl(mo, ScryptedMimeTypes.LocalUrl);
93
93
  }
94
94
  async getPublicLocalEndpoint(nativeId?: ScryptedNativeId): Promise<string> {
@@ -361,9 +361,9 @@ export function attachPluginRemote(peer: RpcPeer, options?: PluginRemoteAttachOp
361
361
 
362
362
  const localStorage = new StorageImpl(deviceManager, undefined);
363
363
 
364
- const remote: PluginRemote & { [PROPERTY_JSON_DISABLE_SERIALIZATION]: boolean, [PROPERTY_PROXY_ONEWAY_METHODS]: string[] } = {
365
- [PROPERTY_JSON_DISABLE_SERIALIZATION]: true,
366
- [PROPERTY_PROXY_ONEWAY_METHODS]: [
364
+ const remote: PluginRemote & { [RpcPeer.PROPERTY_JSON_DISABLE_SERIALIZATION]: boolean, [RpcPeer.PROPERTY_PROXY_ONEWAY_METHODS]: string[] } = {
365
+ [RpcPeer.PROPERTY_JSON_DISABLE_SERIALIZATION]: true,
366
+ [RpcPeer.PROPERTY_PROXY_ONEWAY_METHODS]: [
367
367
  'notify',
368
368
  'updateDeviceState',
369
369
  'setSystemState',
@@ -0,0 +1,49 @@
1
+ import { EventEmitter } from "ws";
2
+ import { RuntimeWorker, RuntimeWorkerOptions } from "./runtime-worker";
3
+ import child_process from 'child_process';
4
+ import { RpcMessage, RpcPeer } from "../../rpc";
5
+
6
+ export abstract class ChildProcessWorker extends EventEmitter implements RuntimeWorker {
7
+ worker: child_process.ChildProcess;
8
+
9
+ constructor(public pluginId: string, options: RuntimeWorkerOptions) {
10
+ super();
11
+ }
12
+
13
+ setupWorker() {
14
+ this.worker.on('close', () => this.emit('close'));
15
+ this.worker.on('disconnect', () => this.emit('disconnect'));
16
+ this.worker.on('exit', (code, signal) => this.emit('exit', code, signal));
17
+ this.worker.on('close', () => this.emit('close'));
18
+ this.worker.on('error', e => this.emit('error', e));
19
+ }
20
+
21
+ get pid() {
22
+ return this.worker.pid;
23
+ }
24
+
25
+ get stdout() {
26
+ return this.worker.stdout;
27
+ }
28
+
29
+ get stderr() {
30
+ return this.worker.stderr;
31
+ }
32
+
33
+ get killed() {
34
+ return this.worker.killed;
35
+ }
36
+
37
+ kill(): void {
38
+ if (!this.worker)
39
+ return;
40
+ this.worker.kill('SIGKILL');
41
+ this.worker.removeAllListeners();
42
+ this.worker.stdout.removeAllListeners();
43
+ this.worker.stderr.removeAllListeners();
44
+ this.worker = undefined;
45
+ }
46
+
47
+ abstract send(message: RpcMessage, reject?: (e: Error) => void): void;
48
+ abstract setupRpcPeer(peer: RpcPeer): void;
49
+ }
@@ -0,0 +1,54 @@
1
+ import { RuntimeWorkerOptions as RuntimeWorkerOptions } from "./runtime-worker";
2
+ import child_process from 'child_process';
3
+ import path from 'path';
4
+ import { RpcMessage, RpcPeer } from "../../rpc";
5
+ import { ChildProcessWorker } from "./child-process-worker";
6
+ import { getPluginNodePath } from "../plugin-npm-dependencies";
7
+
8
+ export class NodeForkWorker extends ChildProcessWorker {
9
+
10
+ constructor(pluginId: string, options: RuntimeWorkerOptions) {
11
+ super(pluginId, options);
12
+
13
+ const {env, pluginDebug} = options;
14
+
15
+ const execArgv: string[] = process.execArgv.slice();
16
+ if (pluginDebug) {
17
+ execArgv.push(`--inspect=0.0.0.0:${pluginDebug.inspectPort}`);
18
+ }
19
+
20
+ this.worker = child_process.fork(require.main.filename, ['child', this.pluginId], {
21
+ stdio: ['pipe', 'pipe', 'pipe', 'ipc'],
22
+ env: Object.assign({}, process.env, env, {
23
+ NODE_PATH: path.join(getPluginNodePath(this.pluginId), 'node_modules'),
24
+ }),
25
+ serialization: 'advanced',
26
+ execArgv,
27
+ });
28
+
29
+ this.setupWorker();
30
+ }
31
+
32
+ setupRpcPeer(peer: RpcPeer): void {
33
+ this.worker.on('message', message => peer.handleMessage(message as any));
34
+ peer.transportSafeArgumentTypes.add(Buffer.name);
35
+ }
36
+
37
+ send(message: RpcMessage, reject?: (e: Error) => void): void {
38
+ try {
39
+ if (!this.worker)
40
+ throw new Error('worked has been killed');
41
+ this.worker.send(message, undefined, e => {
42
+ if (e && reject)
43
+ reject(e);
44
+ });
45
+ }
46
+ catch (e) {
47
+ reject?.(e);
48
+ }
49
+ }
50
+
51
+ get pid() {
52
+ return this.worker.pid;
53
+ }
54
+ }
@@ -0,0 +1,76 @@
1
+ import { EventEmitter } from "ws";
2
+ import { RpcMessage, RpcPeer } from "../../rpc";
3
+ import { RuntimeWorker, RuntimeWorkerOptions } from "./runtime-worker";
4
+ import worker_threads from "worker_threads";
5
+ import path from 'path';
6
+ import { getPluginNodePath } from "../plugin-npm-dependencies";
7
+ import v8 from 'v8';
8
+
9
+ export class NodeThreadWorker extends EventEmitter implements RuntimeWorker {
10
+ terminated: boolean;
11
+ worker: worker_threads.Worker;
12
+
13
+ constructor(public pluginId: string, options: RuntimeWorkerOptions) {
14
+ super();
15
+ const { env } = options;
16
+
17
+ this.worker = new worker_threads.Worker(require.main.filename, {
18
+ argv: ['child-thread', this.pluginId],
19
+ env: Object.assign({}, process.env, env, {
20
+ NODE_PATH: path.join(getPluginNodePath(this.pluginId), 'node_modules'),
21
+ }),
22
+ });
23
+
24
+ this.worker.on('exit', () => {
25
+ this.terminated = true;
26
+ this.emit('exit');
27
+ });
28
+ this.worker.on('error', e => {
29
+ this.emit('error', e);
30
+ });
31
+ this.worker.on('messageerror', e => {
32
+ this.emit('error', e);
33
+ });
34
+ }
35
+
36
+ get pid() {
37
+ return this.worker.threadId;
38
+ }
39
+
40
+ get stdout() {
41
+ return this.worker.stdout;
42
+ }
43
+
44
+ get stderr() {
45
+ return this.worker.stderr;
46
+ }
47
+
48
+ get killed() {
49
+ return this.terminated;
50
+ }
51
+
52
+ kill(): void {
53
+ if (!this.worker)
54
+ return;
55
+ this.worker.terminate();
56
+ this.worker.removeAllListeners();
57
+ this.worker.stdout.removeAllListeners();
58
+ this.worker.stderr.removeAllListeners();
59
+ this.worker = undefined;
60
+ }
61
+
62
+ send(message: RpcMessage, reject?: (e: Error) => void): void {
63
+ try {
64
+ if (!this.worker)
65
+ throw new Error('worked has been killed');
66
+ this.worker.postMessage(v8.serialize(message));
67
+ }
68
+ catch (e) {
69
+ reject?.(e);
70
+ }
71
+ }
72
+
73
+ setupRpcPeer(peer: RpcPeer): void {
74
+ this.worker.on('message', message => peer.handleMessage(v8.deserialize(message)));
75
+ }
76
+ }
@@ -0,0 +1,67 @@
1
+ import { RuntimeWorker, RuntimeWorkerOptions as RuntimeWorkerOptions } from "./runtime-worker";
2
+ import child_process from 'child_process';
3
+ import path from 'path';
4
+ import { EventEmitter } from "ws";
5
+ import { Writable, Readable } from 'stream';
6
+ import { RpcMessage, RpcPeer } from "../../rpc";
7
+ import readline from 'readline';
8
+ import { ChildProcessWorker } from "./child-process-worker";
9
+
10
+ export class PythonRuntimeWorker extends ChildProcessWorker {
11
+
12
+ constructor(pluginId: string, options: RuntimeWorkerOptions) {
13
+ super(pluginId, options);
14
+
15
+ const { env, pluginDebug } = options;
16
+ const args: string[] = [
17
+ '-u',
18
+ ];
19
+ if (pluginDebug) {
20
+ args.push(
21
+ '-m',
22
+ 'debugpy',
23
+ '--listen',
24
+ `0.0.0.0:${pluginDebug.inspectPort}`,
25
+ '--wait-for-client',
26
+ )
27
+ }
28
+ args.push(
29
+ path.join(__dirname, '../../../python', 'plugin-remote.py'),
30
+ )
31
+
32
+ this.worker = child_process.spawn('python3', args, {
33
+ // stdin, stdout, stderr, peer in, peer out
34
+ stdio: ['pipe', 'pipe', 'pipe', 'pipe', 'pipe'],
35
+ env: Object.assign({
36
+ PYTHONPATH: path.join(process.cwd(), 'node_modules/@scrypted/types'),
37
+ }, process.env, env),
38
+ });
39
+
40
+ this.setupWorker();
41
+ }
42
+
43
+ setupRpcPeer(peer: RpcPeer): void {
44
+ const peerin = this.worker.stdio[3] as Writable;
45
+ const peerout = this.worker.stdio[4] as Readable;
46
+
47
+ peerin.on('error', e => this.emit('error', e));
48
+ peerout.on('error', e => this.emit('error', e));
49
+
50
+ const readInterface = readline.createInterface({
51
+ input: peerout,
52
+ terminal: false,
53
+ });
54
+ readInterface.on('line', line => peer.handleMessage(JSON.parse(line)));
55
+ }
56
+
57
+ send(message: RpcMessage, reject?: (e: Error) => void): void {
58
+ try {
59
+ if (!this.worker)
60
+ throw new Error('worked has been killed');
61
+ (this.worker.stdio[3] as Writable).write(JSON.stringify(message) + '\n', e => e && reject?.(e));
62
+ }
63
+ catch (e) {
64
+ reject?.(e);
65
+ }
66
+ }
67
+ }
@@ -0,0 +1,28 @@
1
+ import { RpcMessage, RpcPeer } from "../../rpc";
2
+ import { PluginDebug } from "../plugin-debug";
3
+ import {Readable} from "stream";
4
+
5
+ export interface RuntimeWorkerOptions {
6
+ pluginDebug: PluginDebug;
7
+ env: any;
8
+ }
9
+
10
+ export interface RuntimeWorker {
11
+ pid: number;
12
+ stdout: Readable;
13
+ stderr: Readable;
14
+ killed: boolean;
15
+
16
+ kill(): void;
17
+
18
+ on(event: 'error', listener: (err: Error) => void): this;
19
+ on(event: 'close', listener: (code: number | null, signal: NodeJS.Signals | null) => void): this;
20
+ on(event: 'disconnect', listener: () => void): this;
21
+ on(event: 'exit', listener: (code: number | null, signal: NodeJS.Signals | null) => void): this;
22
+ once(event: 'exit', listener: (code: number | null, signal: NodeJS.Signals | null) => void): this;
23
+
24
+ send(message: RpcMessage, reject?: (e: Error) => void): void;
25
+
26
+ setupRpcPeer(peer: RpcPeer): void;
27
+ }
28
+