@scrypted/server 0.1.13 → 0.1.16
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of @scrypted/server might be problematic. Click here for more details.
- package/dist/http-interfaces.js +11 -0
- package/dist/http-interfaces.js.map +1 -1
- package/dist/plugin/media.js +57 -34
- package/dist/plugin/media.js.map +1 -1
- package/dist/plugin/plugin-host.js +6 -4
- package/dist/plugin/plugin-host.js.map +1 -1
- package/dist/plugin/plugin-http.js +1 -1
- package/dist/plugin/plugin-http.js.map +1 -1
- package/dist/plugin/plugin-npm-dependencies.js +5 -1
- package/dist/plugin/plugin-npm-dependencies.js.map +1 -1
- package/dist/plugin/plugin-remote-worker.js +118 -7
- package/dist/plugin/plugin-remote-worker.js.map +1 -1
- package/dist/plugin/plugin-remote.js +11 -81
- package/dist/plugin/plugin-remote.js.map +1 -1
- package/dist/plugin/runtime/node-fork-worker.js +11 -3
- package/dist/plugin/runtime/node-fork-worker.js.map +1 -1
- package/dist/plugin/runtime/python-worker.js +2 -1
- package/dist/plugin/runtime/python-worker.js.map +1 -1
- package/dist/plugin/socket-serializer.js +17 -0
- package/dist/plugin/socket-serializer.js.map +1 -0
- package/dist/rpc-serializer.js +23 -10
- package/dist/rpc-serializer.js.map +1 -1
- package/dist/rpc.js +1 -1
- package/dist/rpc.js.map +1 -1
- package/dist/runtime.js +26 -24
- package/dist/runtime.js.map +1 -1
- package/dist/scrypted-plugin-main.js +4 -1
- package/dist/scrypted-plugin-main.js.map +1 -1
- package/dist/scrypted-server-main.js +45 -8
- package/dist/scrypted-server-main.js.map +1 -1
- package/package.json +3 -3
- package/python/plugin-remote.py +28 -19
- package/src/http-interfaces.ts +13 -0
- package/src/plugin/media.ts +66 -34
- package/src/plugin/plugin-api.ts +1 -0
- package/src/plugin/plugin-host.ts +6 -4
- package/src/plugin/plugin-http.ts +2 -2
- package/src/plugin/plugin-npm-dependencies.ts +5 -1
- package/src/plugin/plugin-remote-worker.ts +138 -10
- package/src/plugin/plugin-remote.ts +14 -89
- package/src/plugin/runtime/node-fork-worker.ts +11 -3
- package/src/plugin/runtime/python-worker.ts +3 -1
- package/src/plugin/runtime/runtime-worker.ts +1 -1
- package/src/plugin/socket-serializer.ts +15 -0
- package/src/rpc-serializer.ts +30 -13
- package/src/rpc.ts +1 -1
- package/src/runtime.ts +32 -30
- package/src/scrypted-plugin-main.ts +4 -1
- package/src/scrypted-server-main.ts +51 -9
package/python/plugin-remote.py
CHANGED
@@ -6,7 +6,6 @@ import gc
|
|
6
6
|
import json
|
7
7
|
import os
|
8
8
|
import platform
|
9
|
-
import resource
|
10
9
|
import shutil
|
11
10
|
import subprocess
|
12
11
|
import threading
|
@@ -21,7 +20,6 @@ from os import sys
|
|
21
20
|
from typing import Any, Optional, Set, Tuple
|
22
21
|
|
23
22
|
import aiofiles
|
24
|
-
import gi
|
25
23
|
import scrypted_python.scrypted_sdk.types
|
26
24
|
from scrypted_python.scrypted_sdk.types import (Device, DeviceManifest,
|
27
25
|
MediaManager,
|
@@ -31,12 +29,6 @@ from typing_extensions import TypedDict
|
|
31
29
|
|
32
30
|
import rpc
|
33
31
|
|
34
|
-
gi.require_version('Gst', '1.0')
|
35
|
-
|
36
|
-
from gi.repository import GLib, Gst
|
37
|
-
|
38
|
-
Gst.init(None)
|
39
|
-
|
40
32
|
class SystemDeviceState(TypedDict):
|
41
33
|
lastEventTime: int
|
42
34
|
stateTime: int
|
@@ -227,9 +219,9 @@ class PluginRemote:
|
|
227
219
|
if not os.path.exists(python_prefix):
|
228
220
|
os.makedirs(python_prefix)
|
229
221
|
|
230
|
-
|
222
|
+
python_version = 'python%s' % str(
|
231
223
|
sys.version_info[0])+"."+str(sys.version_info[1])
|
232
|
-
print('python:',
|
224
|
+
print('python version:', python_version)
|
233
225
|
|
234
226
|
if 'requirements.txt' in zip.namelist():
|
235
227
|
requirements = zip.open('requirements.txt').read()
|
@@ -254,7 +246,7 @@ class PluginRemote:
|
|
254
246
|
f.write(requirements)
|
255
247
|
f.close()
|
256
248
|
|
257
|
-
p = subprocess.Popen([
|
249
|
+
p = subprocess.Popen([sys.executable, '-m', 'pip', 'install', '-r', requirementstxt,
|
258
250
|
'--prefix', python_prefix], stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
|
259
251
|
while True:
|
260
252
|
line = p.stdout.readline()
|
@@ -275,8 +267,13 @@ class PluginRemote:
|
|
275
267
|
print(str_requirements)
|
276
268
|
|
277
269
|
sys.path.insert(0, zipPath)
|
278
|
-
|
279
|
-
|
270
|
+
if platform.system() != 'Windows':
|
271
|
+
site_packages = os.path.join(
|
272
|
+
python_prefix, 'lib', python_version, 'site-packages')
|
273
|
+
else:
|
274
|
+
site_packages = os.path.join(
|
275
|
+
python_prefix, 'Lib', 'site-packages')
|
276
|
+
print('site-packages: %s' % site_packages)
|
280
277
|
sys.path.insert(0, site_packages)
|
281
278
|
from scrypted_sdk import sdk_init # type: ignore
|
282
279
|
self.systemManager = SystemManager(self.api, self.systemState)
|
@@ -366,7 +363,11 @@ async def async_main(loop: AbstractEventLoop):
|
|
366
363
|
|
367
364
|
def stats_runner():
|
368
365
|
ptime = round(time.process_time() * 1000000)
|
369
|
-
|
366
|
+
try:
|
367
|
+
import resource
|
368
|
+
heapTotal = resource.getrusage(resource.RUSAGE_SELF).ru_maxrss
|
369
|
+
except:
|
370
|
+
heapTotal = 0
|
370
371
|
stats = {
|
371
372
|
'type': 'stats',
|
372
373
|
'cpuUsage': {
|
@@ -399,8 +400,16 @@ def main():
|
|
399
400
|
|
400
401
|
|
401
402
|
if __name__ == "__main__":
|
402
|
-
|
403
|
-
|
404
|
-
|
405
|
-
|
406
|
-
|
403
|
+
try:
|
404
|
+
import gi
|
405
|
+
gi.require_version('Gst', '1.0')
|
406
|
+
from gi.repository import GLib, Gst
|
407
|
+
Gst.init(None)
|
408
|
+
|
409
|
+
worker = threading.Thread(target=main)
|
410
|
+
worker.start()
|
411
|
+
|
412
|
+
loop = GLib.MainLoop()
|
413
|
+
loop.run()
|
414
|
+
except:
|
415
|
+
main()
|
package/src/http-interfaces.ts
CHANGED
@@ -3,6 +3,7 @@ import { Response } from "express";
|
|
3
3
|
import { RpcPeer } from "./rpc";
|
4
4
|
import { join as pathJoin } from 'path';
|
5
5
|
import fs from 'fs';
|
6
|
+
import net from 'net';
|
6
7
|
|
7
8
|
const mime = require('mime/lite');
|
8
9
|
|
@@ -11,6 +12,7 @@ export function createResponseInterface(res: Response, unzippedDir: string, file
|
|
11
12
|
[RpcPeer.PROPERTY_PROXY_ONEWAY_METHODS] = [
|
12
13
|
'send',
|
13
14
|
'sendFile',
|
15
|
+
'sendSocket',
|
14
16
|
];
|
15
17
|
|
16
18
|
send(body: string): void;
|
@@ -61,6 +63,17 @@ export function createResponseInterface(res: Response, unzippedDir: string, file
|
|
61
63
|
}
|
62
64
|
res.sendFile(filePath);
|
63
65
|
}
|
66
|
+
|
67
|
+
sendSocket(socket: net.Socket, options: HttpResponseOptions) {
|
68
|
+
if (options?.code)
|
69
|
+
res.status(options.code);
|
70
|
+
if (options?.headers) {
|
71
|
+
for (const header of Object.keys(options.headers)) {
|
72
|
+
res.setHeader(header, (options.headers as any)[header]);
|
73
|
+
}
|
74
|
+
}
|
75
|
+
socket.pipe(res);
|
76
|
+
}
|
64
77
|
}
|
65
78
|
|
66
79
|
return new HttpResponseImpl();
|
package/src/plugin/media.ts
CHANGED
@@ -1,8 +1,8 @@
|
|
1
|
-
import pathToFfmpeg from 'ffmpeg-static';
|
2
1
|
import { BufferConverter, BufferConvertorOptions, DeviceManager, FFmpegInput, MediaManager, MediaObject, MediaObjectOptions, MediaStreamUrl, ScryptedInterface, ScryptedInterfaceProperty, ScryptedMimeTypes, ScryptedNativeId, SystemDeviceState, SystemManager } from "@scrypted/types";
|
3
2
|
import axios from 'axios';
|
4
3
|
import child_process from 'child_process';
|
5
4
|
import { once } from 'events';
|
5
|
+
import pathToFfmpeg from 'ffmpeg-static';
|
6
6
|
import fs from 'fs';
|
7
7
|
import https from 'https';
|
8
8
|
import mimeType from 'mime';
|
@@ -10,9 +10,8 @@ import mkdirp from "mkdirp";
|
|
10
10
|
import Graph from 'node-dijkstra';
|
11
11
|
import os from 'os';
|
12
12
|
import path from 'path';
|
13
|
-
import rimraf from "rimraf";
|
14
|
-
import tmp from 'tmp';
|
15
13
|
import MimeType from 'whatwg-mimetype';
|
14
|
+
import { safeKillFFmpeg } from '../media-helpers';
|
16
15
|
import { MediaObjectRemote } from "./plugin-api";
|
17
16
|
|
18
17
|
function typeMatches(target: string, candidate: string): boolean {
|
@@ -32,6 +31,7 @@ const httpsAgent = new https.Agent({
|
|
32
31
|
|
33
32
|
export abstract class MediaManagerBase implements MediaManager {
|
34
33
|
builtinConverters: BufferConverter[] = [];
|
34
|
+
extraConverters: BufferConverter[] = [];
|
35
35
|
|
36
36
|
constructor() {
|
37
37
|
for (const h of ['http', 'https']) {
|
@@ -125,40 +125,59 @@ export abstract class MediaManagerBase implements MediaManager {
|
|
125
125
|
convert: async (data, fromMimeType: string, toMimeType: string, options?: BufferConvertorOptions): Promise<Buffer> => {
|
126
126
|
const console = this.getMixinConsole(options?.sourceId, undefined);
|
127
127
|
|
128
|
+
const mt = new MimeType(toMimeType);
|
129
|
+
|
128
130
|
const ffInput: FFmpegInput = JSON.parse(data.toString());
|
129
131
|
|
130
132
|
const args = [
|
131
133
|
'-hide_banner',
|
134
|
+
'-y',
|
132
135
|
];
|
133
136
|
args.push(...ffInput.inputArguments);
|
134
137
|
|
135
|
-
const
|
136
|
-
|
137
|
-
|
138
|
-
|
139
|
-
|
140
|
-
|
141
|
-
|
142
|
-
cp.on('error', (code) => {
|
143
|
-
console.error('ffmpeg error code', code);
|
144
|
-
})
|
145
|
-
const to = setTimeout(() => {
|
146
|
-
console.log('ffmpeg stream to image convesion timed out.');
|
147
|
-
cp.kill('SIGKILL');
|
148
|
-
}, 10000);
|
149
|
-
clearTimeout(to);
|
150
|
-
const [exitCode] = await once(cp, 'exit');
|
151
|
-
if (exitCode)
|
152
|
-
throw new Error(`ffmpeg stream to image convesion failed with exit code: ${exitCode}`);
|
153
|
-
return fs.readFileSync(tmpfile.name);
|
154
|
-
}
|
155
|
-
finally {
|
156
|
-
rimraf.sync(tmpfile.name);
|
138
|
+
const width = parseInt(mt.parameters.get('width'));
|
139
|
+
const height = parseInt(mt.parameters.get('height'));
|
140
|
+
|
141
|
+
if (mt.parameters.get('width') || mt.parameters.get('height')) {
|
142
|
+
args.push(
|
143
|
+
'-vf', `scale=${width || -1}:${height || -1}`,
|
144
|
+
);
|
157
145
|
}
|
146
|
+
|
147
|
+
args.push("-vframes", "1", '-f', 'image2', 'pipe:3');
|
148
|
+
|
149
|
+
const buffers: Buffer[] = [];
|
150
|
+
|
151
|
+
const cp = child_process.spawn(await this.getFFmpegPath(), args, {
|
152
|
+
stdio: ['pipe', 'pipe', 'pipe', 'pipe'],
|
153
|
+
});
|
154
|
+
console.log('converting ffmpeg input to image.');
|
155
|
+
// ffmpegLogInitialOutput(console, cp);
|
156
|
+
cp.on('error', (code) => {
|
157
|
+
console.error('ffmpeg error code', code);
|
158
|
+
});
|
159
|
+
cp.stdio[3].on('data', data => buffers.push(data));
|
160
|
+
const to = setTimeout(() => {
|
161
|
+
console.log('ffmpeg stream to image convesion timed out.');
|
162
|
+
safeKillFFmpeg(cp);
|
163
|
+
}, 10000);
|
164
|
+
const [exitCode] = await once(cp, 'exit');
|
165
|
+
clearTimeout(to);
|
166
|
+
if (exitCode)
|
167
|
+
throw new Error(`ffmpeg stream to image convesion failed with exit code: ${exitCode}`);
|
168
|
+
return Buffer.concat(buffers);
|
158
169
|
}
|
159
170
|
});
|
160
171
|
}
|
161
172
|
|
173
|
+
async addConverter(converter: BufferConverter): Promise<void> {
|
174
|
+
this.extraConverters.push(converter);
|
175
|
+
}
|
176
|
+
|
177
|
+
async clearConverters(): Promise<void> {
|
178
|
+
this.extraConverters = [];
|
179
|
+
}
|
180
|
+
|
162
181
|
async convertMediaObjectToJSON<T>(mediaObject: MediaObject, toMimeType: string): Promise<T> {
|
163
182
|
const buffer = await this.convertMediaObjectToBuffer(mediaObject, toMimeType);
|
164
183
|
return JSON.parse(buffer.toString());
|
@@ -204,7 +223,14 @@ export abstract class MediaManagerBase implements MediaManager {
|
|
204
223
|
const converters = Object.entries(this.getSystemState())
|
205
224
|
.filter(([id, state]) => state[ScryptedInterfaceProperty.interfaces]?.value?.includes(ScryptedInterface.BufferConverter))
|
206
225
|
.map(([id]) => this.getDeviceById<BufferConverter>(id));
|
226
|
+
|
227
|
+
// builtins should be after system converters. these should not be overriden by system,
|
228
|
+
// as it could cause system instability with misconfiguration.
|
207
229
|
converters.push(...this.builtinConverters);
|
230
|
+
|
231
|
+
// extra converters are added last and do allow overriding builtins, as
|
232
|
+
// the instability would be confined to a single plugin.
|
233
|
+
converters.push(...this.extraConverters);
|
208
234
|
return converters;
|
209
235
|
}
|
210
236
|
|
@@ -352,14 +378,16 @@ export abstract class MediaManagerBase implements MediaManager {
|
|
352
378
|
|
353
379
|
// edge matches
|
354
380
|
if (mimeMatches(mediaMime, inputMime)) {
|
381
|
+
const weight = parseFloat(inputMime.parameters.get('converter-weight'));
|
355
382
|
// catch all converters should be heavily weighted so as not to use them.
|
356
|
-
mediaNode[targetId] = inputMime.essence === '*/*' ? 1000 : 1;
|
383
|
+
mediaNode[targetId] = weight || (inputMime.essence === '*/*' ? 1000 : 1);
|
357
384
|
}
|
358
385
|
|
359
386
|
// target output matches
|
360
387
|
if (mimeMatches(outputMime, convertedMime) || converter.toMimeType === ScryptedMimeTypes.MediaObject) {
|
388
|
+
const weight = parseFloat(inputMime.parameters.get('converter-weight'));
|
361
389
|
// catch all converters should be heavily weighted so as not to use them.
|
362
|
-
node['output'] =
|
390
|
+
node['output'] = weight || (convertedMime.essence === ScryptedMimeTypes.MediaObject ? 1000 : 1);
|
363
391
|
}
|
364
392
|
}
|
365
393
|
catch (e) {
|
@@ -382,13 +410,21 @@ export abstract class MediaManagerBase implements MediaManager {
|
|
382
410
|
let value = await mediaObject.getData();
|
383
411
|
let valueMime = new MimeType(mediaObject.mimeType);
|
384
412
|
|
385
|
-
|
413
|
+
while (route.length) {
|
414
|
+
const node = route.shift();
|
386
415
|
const converter = converterReverseids.get(node);
|
387
416
|
const converterToMimeType = new MimeType(converter.toMimeType);
|
388
417
|
const converterFromMimeType = new MimeType(converter.fromMimeType);
|
389
418
|
const type = converterToMimeType.type === '*' ? valueMime.type : converterToMimeType.type;
|
390
419
|
const subtype = converterToMimeType.subtype === '*' ? valueMime.subtype : converterToMimeType.subtype;
|
391
|
-
|
420
|
+
let targetMimeType = `${type}/${subtype}`;
|
421
|
+
if (!route.length && outputMime.parameters.size) {
|
422
|
+
const withParameters = new MimeType(targetMimeType);
|
423
|
+
for (const k of outputMime.parameters.keys()) {
|
424
|
+
withParameters.parameters.set(k, outputMime.parameters.get(k));
|
425
|
+
}
|
426
|
+
targetMimeType = outputMime.toString();
|
427
|
+
}
|
392
428
|
|
393
429
|
if (converter.toMimeType === ScryptedMimeTypes.MediaObject) {
|
394
430
|
const mo = await converter.convert(value, valueMime.essence, toMimeType, { sourceId }) as MediaObject;
|
@@ -436,16 +472,12 @@ export class MediaManagerImpl extends MediaManagerBase {
|
|
436
472
|
|
437
473
|
export class MediaManagerHostImpl extends MediaManagerBase {
|
438
474
|
constructor(public pluginDeviceId: string,
|
439
|
-
public
|
475
|
+
public getSystemState: () => { [id: string]: { [property: string]: SystemDeviceState } },
|
440
476
|
public console: Console,
|
441
477
|
public getDeviceById: (id: string) => any) {
|
442
478
|
super();
|
443
479
|
}
|
444
480
|
|
445
|
-
getSystemState(): { [id: string]: { [property: string]: SystemDeviceState; }; } {
|
446
|
-
return this.systemState;
|
447
|
-
}
|
448
|
-
|
449
481
|
getPluginDeviceId(): string {
|
450
482
|
return this.pluginDeviceId;
|
451
483
|
}
|
package/src/plugin/plugin-api.ts
CHANGED
@@ -135,7 +135,7 @@ export class PluginHost {
|
|
135
135
|
this.io.on('connection', async (socket) => {
|
136
136
|
try {
|
137
137
|
try {
|
138
|
-
if (socket.request.url.indexOf('/api') !== -1) {
|
138
|
+
if (socket.request.url.indexOf('/engine.io/api') !== -1) {
|
139
139
|
if (socket.request.url.indexOf('/public') !== -1) {
|
140
140
|
socket.close();
|
141
141
|
return;
|
@@ -179,7 +179,7 @@ export class PluginHost {
|
|
179
179
|
|
180
180
|
const { runtime } = this.packageJson.scrypted;
|
181
181
|
const mediaManager = runtime === 'python'
|
182
|
-
? new MediaManagerHostImpl(pluginDeviceId, scrypted.stateManager.getSystemState(), console, id => scrypted.getDevice(id))
|
182
|
+
? new MediaManagerHostImpl(pluginDeviceId, () => scrypted.stateManager.getSystemState(), console, id => scrypted.getDevice(id))
|
183
183
|
: undefined;
|
184
184
|
|
185
185
|
this.api = new PluginHostAPI(scrypted, this.pluginId, this, mediaManager);
|
@@ -374,7 +374,8 @@ export class PluginHost {
|
|
374
374
|
serializer.setupRpcPeer(rpcPeer);
|
375
375
|
|
376
376
|
// wrap the host api with a connection specific api that can be torn down on disconnect
|
377
|
-
const
|
377
|
+
const createMediaManager = await this.peer.getParam('createMediaManager');
|
378
|
+
const api = new PluginAPIProxy(this.api, await createMediaManager());
|
378
379
|
const kill = () => {
|
379
380
|
serializer.onDisconnected();
|
380
381
|
api.removeListeners();
|
@@ -389,7 +390,8 @@ export class PluginHost {
|
|
389
390
|
const rpcPeer = createDuplexRpcPeer(`api/${this.pluginId}`, 'duplex', duplex, duplex);
|
390
391
|
|
391
392
|
// wrap the host api with a connection specific api that can be torn down on disconnect
|
392
|
-
const
|
393
|
+
const createMediaManager = await this.peer.getParam('createMediaManager');
|
394
|
+
const api = new PluginAPIProxy(this.api, await createMediaManager());
|
393
395
|
const kill = () => {
|
394
396
|
api.removeListeners();
|
395
397
|
};
|
@@ -38,7 +38,7 @@ export abstract class PluginHttp<T> {
|
|
38
38
|
|
39
39
|
abstract handleEngineIOEndpoint(req: Request, res: ServerResponse, endpointRequest: HttpRequest, pluginData: T): void;
|
40
40
|
abstract handleRequestEndpoint(req: Request, res: Response, endpointRequest: HttpRequest, pluginData: T): void;
|
41
|
-
abstract getEndpointPluginData(endpoint: string, isUpgrade: boolean, isEngineIOEndpoint: boolean): Promise<T>;
|
41
|
+
abstract getEndpointPluginData(req: Request, endpoint: string, isUpgrade: boolean, isEngineIOEndpoint: boolean): Promise<T>;
|
42
42
|
abstract handleWebSocket(endpoint: string, httpRequest: HttpRequest, ws: WebSocket, pluginData: T): Promise<void>;
|
43
43
|
|
44
44
|
async endpointHandler(req: Request, res: Response, isPublicEndpoint: boolean, isEngineIOEndpoint: boolean,
|
@@ -75,7 +75,7 @@ export abstract class PluginHttp<T> {
|
|
75
75
|
return;
|
76
76
|
}
|
77
77
|
|
78
|
-
const pluginData = await this.getEndpointPluginData(endpoint, isUpgrade, isEngineIOEndpoint);
|
78
|
+
const pluginData = await this.getEndpointPluginData(req, endpoint, isUpgrade, isEngineIOEndpoint);
|
79
79
|
if (!pluginData) {
|
80
80
|
end(404, 'Not Found');
|
81
81
|
return;
|
@@ -6,6 +6,7 @@ import { once } from 'events';
|
|
6
6
|
import process from 'process';
|
7
7
|
import mkdirp from "mkdirp";
|
8
8
|
import semver from 'semver';
|
9
|
+
import os from 'os';
|
9
10
|
|
10
11
|
export function getPluginNodePath(name: string) {
|
11
12
|
const pluginVolume = ensurePluginVolume(name);
|
@@ -48,7 +49,10 @@ export async function installOptionalDependencies(console: Console, packageJson:
|
|
48
49
|
mkdirp.sync(nodePrefix);
|
49
50
|
fs.writeFileSync(packageJsonPath, JSON.stringify(reduced));
|
50
51
|
|
51
|
-
|
52
|
+
let npm = 'npm';
|
53
|
+
if (os.platform() === 'win32')
|
54
|
+
npm += '.cmd';
|
55
|
+
const cp = child_process.spawn(npm, ['--prefix', nodePrefix, 'install'], {
|
52
56
|
cwd: nodePrefix,
|
53
57
|
stdio: 'inherit',
|
54
58
|
});
|
@@ -1,16 +1,22 @@
|
|
1
1
|
import { DeviceManager, ScryptedNativeId, ScryptedStatic, SystemManager } from '@scrypted/types';
|
2
|
+
import AdmZip from 'adm-zip';
|
2
3
|
import { Console } from 'console';
|
4
|
+
import fs from 'fs';
|
5
|
+
import { Volume } from 'memfs';
|
3
6
|
import net from 'net';
|
7
|
+
import path from 'path';
|
4
8
|
import { install as installSourceMapSupport } from 'source-map-support';
|
5
9
|
import { PassThrough } from 'stream';
|
6
10
|
import { RpcMessage, RpcPeer } from '../rpc';
|
7
11
|
import { MediaManagerImpl } from './media';
|
8
|
-
import { PluginAPI } from './plugin-api';
|
12
|
+
import { PluginAPI, PluginRemoteLoadZipOptions } from './plugin-api';
|
9
13
|
import { installOptionalDependencies } from './plugin-npm-dependencies';
|
10
|
-
import { attachPluginRemote, PluginReader } from './plugin-remote';
|
14
|
+
import { attachPluginRemote, PluginReader, setupPluginRemote } from './plugin-remote';
|
11
15
|
import { createREPLServer } from './plugin-repl';
|
16
|
+
import { NodeThreadWorker } from './runtime/node-thread-worker';
|
17
|
+
const { link } = require('linkfs');
|
12
18
|
|
13
|
-
export function startPluginRemote(pluginId: string, peerSend: (message: RpcMessage, reject?: (e: Error) => void) => void) {
|
19
|
+
export function startPluginRemote(pluginId: string, peerSend: (message: RpcMessage, reject?: (e: Error) => void, serializationContext?: any) => void) {
|
14
20
|
const peer = new RpcPeer('unknown', 'host', peerSend);
|
15
21
|
|
16
22
|
let systemManager: SystemManager;
|
@@ -66,13 +72,18 @@ export function startPluginRemote(pluginId: string, peerSend: (message: RpcMessa
|
|
66
72
|
return pluginsPromise;
|
67
73
|
}
|
68
74
|
|
75
|
+
const deviceConsoles = new Map<string, Console>();
|
69
76
|
const getDeviceConsole = (nativeId?: ScryptedNativeId) => {
|
70
77
|
// the the plugin console is simply the default console
|
71
78
|
// and gets read from stderr/stdout.
|
72
79
|
if (!nativeId)
|
73
80
|
return console;
|
74
81
|
|
75
|
-
|
82
|
+
let ret = deviceConsoles.get(nativeId);
|
83
|
+
if (ret)
|
84
|
+
return ret;
|
85
|
+
|
86
|
+
ret = getConsole(async (stdout, stderr) => {
|
76
87
|
const connect = async () => {
|
77
88
|
const plugins = await getPlugins();
|
78
89
|
const port = await plugins.getRemoteServicePort(peer.selfName, 'console-writer');
|
@@ -93,10 +104,25 @@ export function startPluginRemote(pluginId: string, peerSend: (message: RpcMessa
|
|
93
104
|
};
|
94
105
|
connect();
|
95
106
|
}, undefined, undefined);
|
107
|
+
|
108
|
+
deviceConsoles.set(nativeId, ret);
|
109
|
+
return ret;
|
96
110
|
}
|
97
111
|
|
112
|
+
const mixinConsoles = new Map<string, Map<string, Console>>();
|
113
|
+
|
98
114
|
const getMixinConsole = (mixinId: string, nativeId: ScryptedNativeId) => {
|
99
|
-
|
115
|
+
let nativeIdConsoles = mixinConsoles.get(nativeId);
|
116
|
+
if (!nativeIdConsoles) {
|
117
|
+
nativeIdConsoles = new Map();
|
118
|
+
mixinConsoles.set(nativeId, nativeIdConsoles);
|
119
|
+
}
|
120
|
+
|
121
|
+
let ret = nativeIdConsoles.get(mixinId);
|
122
|
+
if (ret)
|
123
|
+
return ret;
|
124
|
+
|
125
|
+
ret = getConsole(async (stdout, stderr) => {
|
100
126
|
if (!mixinId) {
|
101
127
|
return;
|
102
128
|
}
|
@@ -147,6 +173,9 @@ export function startPluginRemote(pluginId: string, peerSend: (message: RpcMessa
|
|
147
173
|
}
|
148
174
|
tryConnect();
|
149
175
|
}, getDeviceConsole(nativeId), `[${systemManager.getDeviceById(mixinId)?.name}]`);
|
176
|
+
|
177
|
+
nativeIdConsoles.set(mixinId, ret);
|
178
|
+
return ret;
|
150
179
|
}
|
151
180
|
|
152
181
|
peer.getParam('updateStats').then((updateStats: (stats: any) => void) => {
|
@@ -183,10 +212,6 @@ export function startPluginRemote(pluginId: string, peerSend: (message: RpcMessa
|
|
183
212
|
api = _api;
|
184
213
|
peer.selfName = pluginId;
|
185
214
|
},
|
186
|
-
onPluginReady: async (scrypted, params, plugin) => {
|
187
|
-
replPort = createREPLServer(scrypted, params, plugin);
|
188
|
-
postInstallSourceMapSupport(scrypted);
|
189
|
-
},
|
190
215
|
getPluginConsole,
|
191
216
|
getDeviceConsole,
|
192
217
|
getMixinConsole,
|
@@ -198,7 +223,59 @@ export function startPluginRemote(pluginId: string, peerSend: (message: RpcMessa
|
|
198
223
|
}
|
199
224
|
throw new Error(`unknown service ${name}`);
|
200
225
|
},
|
201
|
-
async onLoadZip(
|
226
|
+
async onLoadZip(scrypted: ScryptedStatic, params: any, packageJson: any, zipData: Buffer | string, zipOptions?: PluginRemoteLoadZipOptions) {
|
227
|
+
let volume: any;
|
228
|
+
let pluginReader: PluginReader;
|
229
|
+
if (zipOptions?.unzippedPath && fs.existsSync(zipOptions?.unzippedPath)) {
|
230
|
+
volume = link(fs, ['', path.join(zipOptions.unzippedPath, 'fs')]);
|
231
|
+
pluginReader = name => {
|
232
|
+
const filename = path.join(zipOptions.unzippedPath, name);
|
233
|
+
if (!fs.existsSync(filename))
|
234
|
+
return;
|
235
|
+
return fs.readFileSync(filename);
|
236
|
+
};
|
237
|
+
}
|
238
|
+
else {
|
239
|
+
const admZip = new AdmZip(zipData);
|
240
|
+
volume = new Volume();
|
241
|
+
for (const entry of admZip.getEntries()) {
|
242
|
+
if (entry.isDirectory)
|
243
|
+
continue;
|
244
|
+
if (!entry.entryName.startsWith('fs/'))
|
245
|
+
continue;
|
246
|
+
const name = entry.entryName.substring('fs/'.length);
|
247
|
+
volume.mkdirpSync(path.dirname(name));
|
248
|
+
const data = entry.getData();
|
249
|
+
volume.writeFileSync(name, data);
|
250
|
+
}
|
251
|
+
|
252
|
+
pluginReader = name => {
|
253
|
+
const entry = admZip.getEntry(name);
|
254
|
+
if (!entry)
|
255
|
+
return;
|
256
|
+
return entry.getData();
|
257
|
+
}
|
258
|
+
}
|
259
|
+
zipData = undefined;
|
260
|
+
|
261
|
+
const pluginConsole = getPluginConsole?.();
|
262
|
+
params.console = pluginConsole;
|
263
|
+
params.require = (name: string) => {
|
264
|
+
if (name === 'fakefs' || (name === 'fs' && !packageJson.scrypted.realfs)) {
|
265
|
+
return volume;
|
266
|
+
}
|
267
|
+
if (name === 'realfs') {
|
268
|
+
return require('fs');
|
269
|
+
}
|
270
|
+
const module = require(name);
|
271
|
+
return module;
|
272
|
+
};
|
273
|
+
const window: any = {};
|
274
|
+
const exports: any = window;
|
275
|
+
window.exports = exports;
|
276
|
+
params.window = window;
|
277
|
+
params.exports = exports;
|
278
|
+
|
202
279
|
const entry = pluginReader('main.nodejs.js.map')
|
203
280
|
const map = entry?.toString();
|
204
281
|
|
@@ -234,6 +311,57 @@ export function startPluginRemote(pluginId: string, peerSend: (message: RpcMessa
|
|
234
311
|
};
|
235
312
|
|
236
313
|
await installOptionalDependencies(getPluginConsole(), packageJson);
|
314
|
+
|
315
|
+
const main = pluginReader('main.nodejs.js');
|
316
|
+
pluginReader = undefined;
|
317
|
+
const script = main.toString();
|
318
|
+
|
319
|
+
scrypted.fork = async () => {
|
320
|
+
const ntw = new NodeThreadWorker(pluginId, {
|
321
|
+
env: process.env,
|
322
|
+
pluginDebug: undefined,
|
323
|
+
});
|
324
|
+
const threadPeer = new RpcPeer('main', 'thread', (message, reject) => ntw.send(message, reject));
|
325
|
+
threadPeer.params.updateStats = (stats: any) => {
|
326
|
+
// todo: merge.
|
327
|
+
// this.stats = stats;
|
328
|
+
}
|
329
|
+
ntw.setupRpcPeer(threadPeer);
|
330
|
+
|
331
|
+
const remote = await setupPluginRemote(threadPeer, api, pluginId, () => systemManager.getSystemState());
|
332
|
+
const forkOptions = Object.assign({}, zipOptions);
|
333
|
+
forkOptions.fork = true;
|
334
|
+
return remote.loadZip(packageJson, zipData, forkOptions)
|
335
|
+
}
|
336
|
+
|
337
|
+
try {
|
338
|
+
peer.evalLocal(script, zipOptions?.filename || '/plugin/main.nodejs.js', params);
|
339
|
+
pluginConsole?.log('plugin successfully loaded');
|
340
|
+
|
341
|
+
if (zipOptions?.fork) {
|
342
|
+
const fork = exports.fork;
|
343
|
+
const ret = await fork();
|
344
|
+
ret[RpcPeer.PROPERTY_JSON_DISABLE_SERIALIZATION] = true;
|
345
|
+
return ret;
|
346
|
+
}
|
347
|
+
|
348
|
+
let pluginInstance = exports.default;
|
349
|
+
// support exporting a plugin class, plugin main function,
|
350
|
+
// or a plugin instance
|
351
|
+
if (pluginInstance.toString().startsWith('class '))
|
352
|
+
pluginInstance = new pluginInstance();
|
353
|
+
if (typeof pluginInstance === 'function')
|
354
|
+
pluginInstance = await pluginInstance();
|
355
|
+
|
356
|
+
replPort = createREPLServer(scrypted, params, pluginInstance);
|
357
|
+
postInstallSourceMapSupport(scrypted);
|
358
|
+
|
359
|
+
return pluginInstance;
|
360
|
+
}
|
361
|
+
catch (e) {
|
362
|
+
pluginConsole?.error('plugin failed to start', e);
|
363
|
+
throw e;
|
364
|
+
}
|
237
365
|
}
|
238
366
|
}).then(scrypted => {
|
239
367
|
systemManager = scrypted.systemManager;
|