@scrypted/server 0.123.22 → 0.123.24
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cluster/cluster-setup.js +1 -1
- package/dist/cluster/cluster-setup.js.map +1 -1
- package/dist/plugin/plugin-api.d.ts +1 -3
- package/dist/plugin/plugin-api.js +1 -3
- package/dist/plugin/plugin-api.js.map +1 -1
- package/dist/plugin/plugin-device.js +1 -1
- package/dist/plugin/plugin-device.js.map +1 -1
- package/dist/plugin/plugin-host.d.ts +2 -3
- package/dist/plugin/plugin-host.js +93 -107
- package/dist/plugin/plugin-host.js.map +1 -1
- package/dist/plugin/plugin-lazy-remote.js +1 -0
- package/dist/plugin/plugin-lazy-remote.js.map +1 -1
- package/dist/plugin/plugin-remote-worker.js +3 -13
- package/dist/plugin/plugin-remote-worker.js.map +1 -1
- package/dist/rpc.d.ts +1 -0
- package/dist/rpc.js +2 -0
- package/dist/rpc.js.map +1 -1
- package/dist/scrypted-cluster-main.d.ts +1 -0
- package/dist/scrypted-cluster-main.js +14 -5
- package/dist/scrypted-cluster-main.js.map +1 -1
- package/dist/services/cluster-fork.js +7 -1
- package/dist/services/cluster-fork.js.map +1 -1
- package/dist/services/plugin.d.ts +0 -1
- package/dist/services/plugin.js +0 -1
- package/dist/services/plugin.js.map +1 -1
- package/package.json +1 -1
- package/python/plugin_remote.py +3 -44
- package/src/cluster/cluster-setup.ts +1 -1
- package/src/plugin/plugin-api.ts +2 -3
- package/src/plugin/plugin-device.ts +1 -1
- package/src/plugin/plugin-host.ts +102 -115
- package/src/plugin/plugin-lazy-remote.ts +1 -0
- package/src/plugin/plugin-remote-worker.ts +4 -15
- package/src/rpc.ts +2 -0
- package/src/scrypted-cluster-main.ts +18 -6
- package/src/services/cluster-fork.ts +7 -1
- package/src/services/plugin.ts +0 -1
- package/dist/plugin/plugin-remote-stats.d.ts +0 -7
- package/dist/plugin/plugin-remote-stats.js +0 -36
- package/dist/plugin/plugin-remote-stats.js.map +0 -1
- package/src/plugin/plugin-remote-stats.ts +0 -44
@@ -4,13 +4,13 @@ import * as io from 'engine.io';
|
|
4
4
|
import fs from 'fs';
|
5
5
|
import os from 'os';
|
6
6
|
import WebSocket from 'ws';
|
7
|
+
import { setupCluster } from '../cluster/cluster-setup';
|
7
8
|
import { Plugin } from '../db-types';
|
8
9
|
import { IOServer, IOServerSocket } from '../io';
|
9
10
|
import { Logger } from '../logger';
|
10
11
|
import { RpcPeer, RPCResultError } from '../rpc';
|
11
12
|
import { createRpcSerializer } from '../rpc-serializer';
|
12
13
|
import { ScryptedRuntime } from '../runtime';
|
13
|
-
import { setupCluster } from '../cluster/cluster-setup';
|
14
14
|
import { sleep } from '../sleep';
|
15
15
|
import { AccessControls } from './acl';
|
16
16
|
import { MediaManagerHostImpl } from './media';
|
@@ -20,7 +20,6 @@ import { PluginDebug } from './plugin-debug';
|
|
20
20
|
import { PluginHostAPI } from './plugin-host-api';
|
21
21
|
import { LazyRemote } from './plugin-lazy-remote';
|
22
22
|
import { setupPluginRemote } from './plugin-remote';
|
23
|
-
import { PluginStats } from './plugin-remote-stats';
|
24
23
|
import { WebSocketConnection } from './plugin-remote-websocket';
|
25
24
|
import { ensurePluginVolume, getScryptedVolume } from './plugin-volume';
|
26
25
|
import { createClusterForkWorker, needsClusterForkWorker } from './runtime/cluster-fork.worker';
|
@@ -60,8 +59,6 @@ export class PluginHost {
|
|
60
59
|
api: PluginHostAPI;
|
61
60
|
pluginName: string;
|
62
61
|
packageJson: any;
|
63
|
-
lastStats: number;
|
64
|
-
stats: PluginStats;
|
65
62
|
killed = false;
|
66
63
|
consoleServer: Promise<ConsoleServer>;
|
67
64
|
zipHash: string;
|
@@ -173,7 +170,6 @@ export class PluginHost {
|
|
173
170
|
return;
|
174
171
|
}
|
175
172
|
|
176
|
-
|
177
173
|
const handler = this.scrypted.getDevice<EngineIOHandler>(pluginDevice._id);
|
178
174
|
|
179
175
|
// @ts-expect-error
|
@@ -202,8 +198,6 @@ export class PluginHost {
|
|
202
198
|
}
|
203
199
|
})
|
204
200
|
|
205
|
-
const self = this;
|
206
|
-
|
207
201
|
const { runtime } = this.packageJson.scrypted;
|
208
202
|
const mediaManager = runtime && runtime !== 'node'
|
209
203
|
? new MediaManagerHostImpl(pluginDeviceId, () => scrypted.stateManager.getSystemState(), console, id => scrypted.getDevice(id))
|
@@ -214,79 +208,120 @@ export class PluginHost {
|
|
214
208
|
logger.log('i', `loading ${this.pluginName}`);
|
215
209
|
logger.log('i', 'pid ' + this.worker?.pid);
|
216
210
|
|
217
|
-
const remotePromise = (
|
218
|
-
|
211
|
+
const remotePromise = this.prepareRemote(peerPromise, logger, pluginDebug);
|
212
|
+
const init = this.initializeRemote(remotePromise, logger, pluginDebug);
|
213
|
+
|
214
|
+
init.catch(e => {
|
215
|
+
console.error('plugin failed to load', e);
|
216
|
+
this.api.removeListeners();
|
217
|
+
});
|
218
|
+
|
219
|
+
this.module = init.then(({ module }) => module);
|
220
|
+
const remote = init.then(({ remote }) => remote);
|
221
|
+
this.remote = new LazyRemote(remotePromise, remote);
|
222
|
+
}
|
223
|
+
|
224
|
+
private async initializeRemote(remotePromise: Promise<PluginRemote>, logger: Logger, pluginDebug: PluginDebug) {
|
225
|
+
const remote = await remotePromise;
|
226
|
+
|
227
|
+
await Promise.all(
|
228
|
+
this.scrypted.findPluginDevices(this.pluginId)
|
229
|
+
.map(pluginDevice => remote.setNativeId(pluginDevice.nativeId, pluginDevice._id, pluginDevice.storage || {}))
|
230
|
+
);
|
231
|
+
|
232
|
+
const waitDebug = pluginDebug?.waitDebug;
|
233
|
+
if (waitDebug) {
|
234
|
+
console.info('waiting for debugger...');
|
219
235
|
try {
|
220
|
-
|
236
|
+
await waitDebug;
|
237
|
+
console.info('debugger attached.');
|
238
|
+
await sleep(1000);
|
221
239
|
}
|
222
240
|
catch (e) {
|
223
|
-
|
224
|
-
throw new RPCResultError(this.peer, 'cluster plugin start failed', e);
|
241
|
+
console.error('debugger failed', e);
|
225
242
|
}
|
226
|
-
|
227
|
-
})();
|
243
|
+
}
|
228
244
|
|
229
|
-
const
|
230
|
-
|
245
|
+
const fail = 'Plugin failed to load. View Console for more information.';
|
246
|
+
try {
|
247
|
+
const loadZipOptions: PluginRemoteLoadZipOptions = {
|
248
|
+
clusterId: this.scrypted.clusterId,
|
249
|
+
clusterSecret: this.scrypted.clusterSecret,
|
250
|
+
// debug flag can be used to affect path resolution for sourcemaps etc.
|
251
|
+
debug: !!pluginDebug,
|
252
|
+
zipHash: this.zipHash,
|
253
|
+
};
|
254
|
+
// original implementation sent the zipBuffer, sending the zipFile name now.
|
255
|
+
// can switch back for non-local plugins.
|
256
|
+
const modulePromise = remote.loadZip(this.packageJson,
|
257
|
+
new PluginZipAPI(async () => fs.promises.readFile(this.zipFile)),
|
258
|
+
loadZipOptions);
|
259
|
+
// allow garbage collection of the zip buffer
|
260
|
+
const module = await modulePromise;
|
261
|
+
logger.log('i', `loaded ${this.pluginName}`);
|
262
|
+
logger.clearAlert(fail)
|
263
|
+
return { module, remote };
|
264
|
+
}
|
265
|
+
catch (e) {
|
266
|
+
logger.log('a', fail);
|
267
|
+
logger.log('e', `plugin load error ${e}`);
|
268
|
+
console.error('plugin load error', e);
|
269
|
+
throw e;
|
270
|
+
}
|
271
|
+
}
|
231
272
|
|
232
|
-
|
233
|
-
|
234
|
-
|
235
|
-
|
273
|
+
private async prepareRemote(peerPromise: Promise<RpcPeer>, logger: Logger, pluginDebug: PluginDebug) {
|
274
|
+
let peer: RpcPeer;
|
275
|
+
try {
|
276
|
+
peer = await peerPromise;
|
277
|
+
}
|
278
|
+
catch (e) {
|
279
|
+
logger.log('e', 'plugin failed to start ' + e);
|
280
|
+
throw new RPCResultError(this.peer, 'cluster plugin start failed', e);
|
281
|
+
}
|
236
282
|
|
237
|
-
|
238
|
-
|
239
|
-
console.info('waiting for debugger...');
|
240
|
-
try {
|
241
|
-
await waitDebug;
|
242
|
-
console.info('debugger attached.');
|
243
|
-
await sleep(1000);
|
244
|
-
}
|
245
|
-
catch (e) {
|
246
|
-
console.error('debugger failed', e);
|
247
|
-
}
|
248
|
-
}
|
283
|
+
const startupTime = Date.now();
|
284
|
+
let lastPong: number;
|
249
285
|
|
250
|
-
|
286
|
+
(async () => {
|
251
287
|
try {
|
252
|
-
|
253
|
-
|
254
|
-
|
255
|
-
|
256
|
-
|
257
|
-
|
258
|
-
|
259
|
-
|
260
|
-
|
261
|
-
const modulePromise = remote.loadZip(this.packageJson,
|
262
|
-
// the plugin is expected to send process stats every 10 seconds.
|
263
|
-
// this can be used as a check for liveness.
|
264
|
-
new PluginZipAPI(async () => fs.promises.readFile(this.zipFile), async (stats: PluginStats) => {
|
265
|
-
this.lastStats = Date.now();
|
266
|
-
this.stats = stats;
|
267
|
-
}),
|
268
|
-
loadZipOptions);
|
269
|
-
// allow garbage collection of the zip buffer
|
270
|
-
const module = await modulePromise;
|
271
|
-
logger.log('i', `loaded ${this.pluginName}`);
|
272
|
-
logger.clearAlert(fail)
|
273
|
-
return { module, remote };
|
288
|
+
let pingPromise: Promise<(time: number) => Promise<number>>
|
289
|
+
while (!this.killed) {
|
290
|
+
await sleep(30000);
|
291
|
+
if (this.killed)
|
292
|
+
return;
|
293
|
+
pingPromise ||= peer.getParam('ping');
|
294
|
+
const ping = await pingPromise;
|
295
|
+
lastPong = await ping(Date.now());
|
296
|
+
}
|
274
297
|
}
|
275
298
|
catch (e) {
|
276
|
-
logger.log('
|
277
|
-
|
278
|
-
console.error('plugin load error', e);
|
279
|
-
throw e;
|
299
|
+
logger.log('e', 'plugin ping failed. restarting.');
|
300
|
+
this.api.requestRestart();
|
280
301
|
}
|
281
302
|
})();
|
282
303
|
|
283
|
-
|
284
|
-
|
304
|
+
const healthInterval = setInterval(async () => {
|
305
|
+
const now = Date.now();
|
306
|
+
// plugin may take a while to install, so wait 10 minutes.
|
307
|
+
// after that, require 1 minute checkins.
|
308
|
+
if (!lastPong) {
|
309
|
+
if (now - startupTime > 10 * 60 * 1000) {
|
310
|
+
const logger = await this.api.getLogger(undefined);
|
311
|
+
logger.log('e', 'plugin failed to start in a timely manner. restarting.');
|
312
|
+
this.api.requestRestart();
|
313
|
+
}
|
314
|
+
return;
|
315
|
+
}
|
316
|
+
if (!pluginDebug && (lastPong + 60000 < now)) {
|
317
|
+
const logger = await this.api.getLogger(undefined);
|
318
|
+
logger.log('e', 'plugin is not responding to ping. restarting.');
|
319
|
+
this.api.requestRestart();
|
320
|
+
}
|
321
|
+
}, 60000);
|
322
|
+
peer.killedSafe.finally(() => clearInterval(healthInterval));
|
285
323
|
|
286
|
-
|
287
|
-
console.error('plugin failed to load', e);
|
288
|
-
this.api.removeListeners();
|
289
|
-
});
|
324
|
+
return setupPluginRemote(peer, this.api, this.pluginId, { serverVersion }, () => this.scrypted.stateManager.getSystemState());
|
290
325
|
}
|
291
326
|
|
292
327
|
startPluginHost(logger: Logger, env: any, pluginDebug: PluginDebug) {
|
@@ -355,9 +390,9 @@ export class PluginHost {
|
|
355
390
|
|
356
391
|
forkPeer.then(peer => {
|
357
392
|
const originalPeer = this.peer;
|
358
|
-
originalPeer.
|
393
|
+
originalPeer.killedSafe.finally(() => peer.kill());
|
359
394
|
this.peer = peer;
|
360
|
-
peer.
|
395
|
+
peer.killedSafe.finally(() => originalPeer.kill());
|
361
396
|
}).catch(() => {});
|
362
397
|
|
363
398
|
this.worker = runtimeWorker;
|
@@ -387,54 +422,6 @@ export class PluginHost {
|
|
387
422
|
disconnect();
|
388
423
|
});
|
389
424
|
|
390
|
-
const startupTime = Date.now();
|
391
|
-
let lastPong: number;
|
392
|
-
const pong = (time: number) => {
|
393
|
-
lastPong = time;
|
394
|
-
};
|
395
|
-
(async () => {
|
396
|
-
try {
|
397
|
-
let pingPromise: Promise<(time: number, p: typeof pong) => Promise<void>>
|
398
|
-
while (!this.killed) {
|
399
|
-
await sleep(30000);
|
400
|
-
if (this.killed)
|
401
|
-
return;
|
402
|
-
pingPromise ||= this.peer.getParam('ping');
|
403
|
-
const ping = await pingPromise;
|
404
|
-
await ping(Date.now(), pong);
|
405
|
-
}
|
406
|
-
}
|
407
|
-
catch (e) {
|
408
|
-
logger.log('e', 'plugin ping failed. restarting.');
|
409
|
-
this.api.requestRestart();
|
410
|
-
}
|
411
|
-
})();
|
412
|
-
|
413
|
-
const healthInterval = setInterval(async () => {
|
414
|
-
const now = Date.now();
|
415
|
-
// plugin may take a while to install, so wait 10 minutes.
|
416
|
-
// after that, require 1 minute checkins.
|
417
|
-
if (!this.lastStats || !lastPong) {
|
418
|
-
if (now - startupTime > 10 * 60 * 1000) {
|
419
|
-
const logger = await this.api.getLogger(undefined);
|
420
|
-
logger.log('e', 'plugin failed to start in a timely manner. restarting.');
|
421
|
-
this.api.requestRestart();
|
422
|
-
}
|
423
|
-
return;
|
424
|
-
}
|
425
|
-
if (!pluginDebug && (this.lastStats + 60000 < now)) {
|
426
|
-
const logger = await this.api.getLogger(undefined);
|
427
|
-
logger.log('e', 'plugin is not reporting stats. restarting.');
|
428
|
-
this.api.requestRestart();
|
429
|
-
}
|
430
|
-
if (!pluginDebug && (lastPong + 60000 < now)) {
|
431
|
-
const logger = await this.api.getLogger(undefined);
|
432
|
-
logger.log('e', 'plugin is not responding to ping. restarting.');
|
433
|
-
this.api.requestRestart();
|
434
|
-
}
|
435
|
-
}, 60000);
|
436
|
-
this.peer.killed.finally(() => clearInterval(healthInterval));
|
437
|
-
|
438
425
|
return peer;
|
439
426
|
}
|
440
427
|
|
@@ -15,6 +15,7 @@ import { PluginRemote, PluginRemoteLoadZipOptions, PluginZipAPI } from './plugin
|
|
15
15
|
this.remote = await remoteReadyPromise;
|
16
16
|
return this.remote;
|
17
17
|
})();
|
18
|
+
this.remoteReadyPromise.catch(() => {});
|
18
19
|
}
|
19
20
|
|
20
21
|
async loadZip(packageJson: any, zipAPI: PluginZipAPI, options?: PluginRemoteLoadZipOptions): Promise<any> {
|
@@ -12,7 +12,6 @@ import { PluginAPI, PluginAPIProxy, PluginRemote, PluginRemoteLoadZipOptions, Pl
|
|
12
12
|
import { pipeWorkerConsole, prepareConsoles } from './plugin-console';
|
13
13
|
import { getPluginNodePath, installOptionalDependencies } from './plugin-npm-dependencies';
|
14
14
|
import { attachPluginRemote, DeviceManagerImpl, setupPluginRemote } from './plugin-remote';
|
15
|
-
import { PluginStats, startStatsUpdater } from './plugin-remote-stats';
|
16
15
|
import { createREPLServer } from './plugin-repl';
|
17
16
|
import { getPluginVolume } from './plugin-volume';
|
18
17
|
import { ChildProcessWorker } from './runtime/child-process-worker';
|
@@ -33,7 +32,7 @@ export function startPluginRemote(mainFilename: string, pluginId: string, peerSe
|
|
33
32
|
const peer = new RpcPeer('unknown', 'host', peerSend);
|
34
33
|
|
35
34
|
const clusterPeerSetup = setupCluster(peer);
|
36
|
-
const { initializeCluster, connectRPCObject, mainThreadBrokerRegister
|
35
|
+
const { initializeCluster, connectRPCObject, mainThreadBrokerRegister, mainThreadPort } = clusterPeerSetup;
|
37
36
|
|
38
37
|
peer.params.initializeCluster = initializeCluster;
|
39
38
|
|
@@ -196,14 +195,8 @@ export function startPluginRemote(mainFilename: string, pluginId: string, peerSe
|
|
196
195
|
|
197
196
|
await installOptionalDependencies(getPluginConsole(), packageJson);
|
198
197
|
|
199
|
-
|
200
|
-
|
201
|
-
const allMemoryStats = new Map<RuntimeWorker, NodeJS.MemoryUsage>();
|
202
|
-
// start the stats updater/watchdog after installation has finished, as that may take some time.
|
203
|
-
startStatsUpdater(allMemoryStats, zipAPI.updateStats);
|
204
|
-
|
205
|
-
peer.params.ping = async (time: number, pong: (time: number) => Promise<void>) => {
|
206
|
-
await pong(time);
|
198
|
+
peer.params.ping = async (time: number) => {
|
199
|
+
return time;
|
207
200
|
};
|
208
201
|
|
209
202
|
const main = pluginReader(mainNodejs);
|
@@ -315,13 +308,11 @@ export function startPluginRemote(mainFilename: string, pluginId: string, peerSe
|
|
315
308
|
threadPeer.kill('worker exited');
|
316
309
|
forkApi.removeListeners();
|
317
310
|
forks.delete(remote);
|
318
|
-
allMemoryStats.delete(runtimeWorker);
|
319
311
|
});
|
320
312
|
runtimeWorker.on('error', e => {
|
321
313
|
threadPeer.kill('worker error ' + e);
|
322
314
|
forkApi.removeListeners();
|
323
315
|
forks.delete(remote);
|
324
|
-
allMemoryStats.delete(runtimeWorker);
|
325
316
|
});
|
326
317
|
|
327
318
|
for (const [nativeId, dmd] of deviceManager.nativeIds.entries()) {
|
@@ -331,9 +322,7 @@ export function startPluginRemote(mainFilename: string, pluginId: string, peerSe
|
|
331
322
|
const forkOptions = Object.assign({}, zipOptions);
|
332
323
|
forkOptions.fork = true;
|
333
324
|
forkOptions.main = options?.filename;
|
334
|
-
const forkZipAPI = new PluginZipAPI(() => zipAPI.getZip()
|
335
|
-
allMemoryStats.set(runtimeWorker, stats.memoryUsage);
|
336
|
-
});
|
325
|
+
const forkZipAPI = new PluginZipAPI(() => zipAPI.getZip());
|
337
326
|
return remote.loadZip(packageJson, forkZipAPI, forkOptions)
|
338
327
|
})();
|
339
328
|
|
package/src/rpc.ts
CHANGED
@@ -300,6 +300,7 @@ export class RpcPeer {
|
|
300
300
|
constructorSerializerMap = new Map<any, string>();
|
301
301
|
transportSafeArgumentTypes = RpcPeer.getDefaultTransportSafeArgumentTypes();
|
302
302
|
killed: Promise<string>;
|
303
|
+
killedSafe: Promise<void>;
|
303
304
|
killedDeferred: Deferred;
|
304
305
|
tags: any = {};
|
305
306
|
yieldedAsyncIterators = new Set<AsyncGenerator>();
|
@@ -395,6 +396,7 @@ export class RpcPeer {
|
|
395
396
|
this.killed = new Promise<string>((resolve, reject) => {
|
396
397
|
this.killedDeferred = { resolve, reject, method: undefined };
|
397
398
|
}).catch(e => e.message || 'Unknown Error');
|
399
|
+
this.killedSafe = this.killed.then(() => {}).catch(() => { });
|
398
400
|
}
|
399
401
|
|
400
402
|
static isTransportSafe(value: any) {
|
@@ -1,3 +1,4 @@
|
|
1
|
+
import os from 'os';
|
1
2
|
import type { ForkOptions } from '@scrypted/types';
|
2
3
|
import { once } from 'events';
|
3
4
|
import net from 'net';
|
@@ -39,7 +40,7 @@ function peerLifecycle(serializer: ReturnType<typeof createRpcDuplexSerializer>,
|
|
39
40
|
socket.on('close', () => {
|
40
41
|
peer.kill(`cluster ${type} closed`);
|
41
42
|
});
|
42
|
-
peer.
|
43
|
+
peer.killedSafe.finally(() => {
|
43
44
|
socket.destroy();
|
44
45
|
});
|
45
46
|
}
|
@@ -68,6 +69,7 @@ export interface ClusterWorkerProperties {
|
|
68
69
|
|
69
70
|
export interface ClusterWorker extends ClusterWorkerProperties {
|
70
71
|
peer: RpcPeer;
|
72
|
+
forks: Set<ClusterForkOptions>;
|
71
73
|
}
|
72
74
|
|
73
75
|
export class PeerLiveness {
|
@@ -120,7 +122,7 @@ export function startClusterClient(mainFilename: string) {
|
|
120
122
|
try {
|
121
123
|
await once(rawSocket, 'connect');
|
122
124
|
}
|
123
|
-
catch(
|
125
|
+
catch (e) {
|
124
126
|
continue;
|
125
127
|
}
|
126
128
|
|
@@ -154,7 +156,7 @@ export function startClusterClient(mainFilename: string) {
|
|
154
156
|
const auth: ClusterObject = {
|
155
157
|
address: socket.localAddress,
|
156
158
|
port: socket.localPort,
|
157
|
-
id:
|
159
|
+
id: process.env.SCRYPTED_CLUSTER_ID || os.hostname(),
|
158
160
|
proxyId: undefined,
|
159
161
|
sourceKey: undefined,
|
160
162
|
sha256: undefined,
|
@@ -208,17 +210,19 @@ export function startClusterClient(mainFilename: string) {
|
|
208
210
|
runtimeWorker.on('error', e => {
|
209
211
|
threadPeer.kill('worker error ' + e);
|
210
212
|
});
|
211
|
-
threadPeer.
|
213
|
+
threadPeer.killedSafe.finally(() => {
|
212
214
|
runtimeWorker.kill();
|
213
215
|
});
|
214
216
|
peerLiveness.waitKilled().catch(() => { }).finally(() => {
|
215
217
|
threadPeer.kill('peer killed');
|
216
218
|
});
|
217
219
|
let getRemote: any;
|
220
|
+
let ping: any;
|
218
221
|
try {
|
219
222
|
const initializeCluster: InitializeCluster = await threadPeer.getParam('initializeCluster');
|
220
223
|
await initializeCluster({ clusterId, clusterSecret });
|
221
224
|
getRemote = await threadPeer.getParam('getRemote');
|
225
|
+
ping = await threadPeer.getParam('ping');
|
222
226
|
}
|
223
227
|
catch (e) {
|
224
228
|
threadPeer.kill('cluster fork failed');
|
@@ -241,6 +245,7 @@ export function startClusterClient(mainFilename: string) {
|
|
241
245
|
stdout: readStream(runtimeWorker.stdout),
|
242
246
|
stderr: readStream(runtimeWorker.stderr),
|
243
247
|
getRemote,
|
248
|
+
ping,
|
244
249
|
};
|
245
250
|
};
|
246
251
|
|
@@ -254,9 +259,12 @@ export function startClusterClient(mainFilename: string) {
|
|
254
259
|
}
|
255
260
|
catch (e) {
|
256
261
|
peer.kill(e.message);
|
257
|
-
socket.destroy();
|
258
262
|
console.warn('Cluster client error:', localAddress, localPort, e);
|
259
263
|
}
|
264
|
+
finally {
|
265
|
+
peer.kill();
|
266
|
+
socket.destroy();
|
267
|
+
}
|
260
268
|
}
|
261
269
|
})();
|
262
270
|
}
|
@@ -278,6 +286,9 @@ export function createClusterServer(runtime: ScryptedRuntime, certificate: Retur
|
|
278
286
|
const sha256 = computeClusterObjectHash(auth, runtime.clusterSecret);
|
279
287
|
if (sha256 !== auth.sha256)
|
280
288
|
throw new Error('cluster object hash mismatch');
|
289
|
+
|
290
|
+
peer.peerName = auth.id || `${socket.remoteAddress}`;
|
291
|
+
|
281
292
|
// the remote address may be ipv6 prefixed so use a fuzzy match.
|
282
293
|
// eg ::ffff:192.168.2.124
|
283
294
|
if (!process.env.SCRYPTED_DISABLE_CLUSTER_SERVER_TRUST) {
|
@@ -287,9 +298,10 @@ export function createClusterServer(runtime: ScryptedRuntime, certificate: Retur
|
|
287
298
|
const worker: ClusterWorker = {
|
288
299
|
...properties,
|
289
300
|
peer,
|
301
|
+
forks: new Set(),
|
290
302
|
};
|
291
303
|
runtime.clusterWorkers.add(worker);
|
292
|
-
peer.
|
304
|
+
peer.killedSafe.finally(() => {
|
293
305
|
runtime.clusterWorkers.delete(worker);
|
294
306
|
});
|
295
307
|
socket.on('close', () => {
|
@@ -18,7 +18,12 @@ export class ClusterFork {
|
|
18
18
|
throw new Error(`no worker found for cluster labels ${JSON.stringify(options.labels)}`);
|
19
19
|
|
20
20
|
const fork: ClusterForkParam = await worker.peer.getParam('fork');
|
21
|
-
|
21
|
+
const forkResult = await fork(peerLiveness, options.runtime, packageJson, zipHash, getZip);
|
22
|
+
worker.forks.add(options);
|
23
|
+
forkResult.waitKilled().catch(() => {}).finally(() => {
|
24
|
+
worker.forks.delete(options);
|
25
|
+
});
|
26
|
+
return forkResult;
|
22
27
|
}
|
23
28
|
|
24
29
|
async getClusterWorkers() {
|
@@ -26,6 +31,7 @@ export class ClusterFork {
|
|
26
31
|
for (const worker of this.runtime.clusterWorkers) {
|
27
32
|
ret[worker.peer.peerName] = {
|
28
33
|
labels: worker.labels,
|
34
|
+
forks: [...worker.forks],
|
29
35
|
};
|
30
36
|
}
|
31
37
|
return ret;
|
package/src/services/plugin.ts
CHANGED
@@ -1,7 +0,0 @@
|
|
1
|
-
import { RuntimeWorker } from "./runtime/runtime-worker";
|
2
|
-
export interface PluginStats {
|
3
|
-
type: 'stats';
|
4
|
-
cpuUsage: NodeJS.CpuUsage;
|
5
|
-
memoryUsage: NodeJS.MemoryUsage;
|
6
|
-
}
|
7
|
-
export declare function startStatsUpdater(allMemoryStats: Map<RuntimeWorker, NodeJS.MemoryUsage>, updateStats: (stats: PluginStats) => Promise<void>): void;
|
@@ -1,36 +0,0 @@
|
|
1
|
-
"use strict";
|
2
|
-
Object.defineProperty(exports, "__esModule", { value: true });
|
3
|
-
exports.startStatsUpdater = startStatsUpdater;
|
4
|
-
function startStatsUpdater(allMemoryStats, updateStats) {
|
5
|
-
if (!updateStats)
|
6
|
-
console.warn('wtf');
|
7
|
-
setInterval(() => {
|
8
|
-
let cpuUsage;
|
9
|
-
let memoryUsage;
|
10
|
-
if (process.cpuUsage)
|
11
|
-
cpuUsage = process.cpuUsage();
|
12
|
-
allMemoryStats.set(undefined, process.memoryUsage());
|
13
|
-
memoryUsage = {
|
14
|
-
rss: 0,
|
15
|
-
heapTotal: 0,
|
16
|
-
heapUsed: 0,
|
17
|
-
external: 0,
|
18
|
-
arrayBuffers: 0,
|
19
|
-
};
|
20
|
-
for (const mu of allMemoryStats.values()) {
|
21
|
-
if (!mu)
|
22
|
-
continue;
|
23
|
-
memoryUsage.rss += mu.rss;
|
24
|
-
memoryUsage.heapTotal += mu.heapTotal;
|
25
|
-
memoryUsage.heapUsed += mu.heapUsed;
|
26
|
-
memoryUsage.external += mu.external;
|
27
|
-
memoryUsage.arrayBuffers += mu.arrayBuffers;
|
28
|
-
}
|
29
|
-
updateStats({
|
30
|
-
type: 'stats',
|
31
|
-
cpuUsage,
|
32
|
-
memoryUsage,
|
33
|
-
}).catch(() => { });
|
34
|
-
}, 10000);
|
35
|
-
}
|
36
|
-
//# sourceMappingURL=plugin-remote-stats.js.map
|
@@ -1 +0,0 @@
|
|
1
|
-
{"version":3,"file":"plugin-remote-stats.js","sourceRoot":"","sources":["../../src/plugin/plugin-remote-stats.ts"],"names":[],"mappings":";;AAQA,8CAmCC;AAnCD,SAAgB,iBAAiB,CAAC,cAAsD,EAAE,WAAkD;IACxI,IAAI,CAAC,WAAW;QACZ,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,CAAA;IACvB,WAAW,CAAC,GAAG,EAAE;QACb,IAAI,QAAyB,CAAC;QAC9B,IAAI,WAA+B,CAAC;QACpC,IAAI,OAAO,CAAC,QAAQ;YAChB,QAAQ,GAAG,OAAO,CAAC,QAAQ,EAAE,CAAC;QAElC,cAAc,CAAC,GAAG,CAAC,SAAS,EAAE,OAAO,CAAC,WAAW,EAAE,CAAC,CAAC;QAErD,WAAW,GAAG;YACV,GAAG,EAAE,CAAC;YACN,SAAS,EAAE,CAAC;YACZ,QAAQ,EAAE,CAAC;YACX,QAAQ,EAAE,CAAC;YACX,YAAY,EAAE,CAAC;SAClB,CAAA;QAED,KAAK,MAAM,EAAE,IAAI,cAAc,CAAC,MAAM,EAAE,EAAE,CAAC;YACvC,IAAI,CAAC,EAAE;gBACH,SAAS;YACb,WAAW,CAAC,GAAG,IAAI,EAAE,CAAC,GAAG,CAAC;YAC1B,WAAW,CAAC,SAAS,IAAI,EAAE,CAAC,SAAS,CAAC;YACtC,WAAW,CAAC,QAAQ,IAAI,EAAE,CAAC,QAAQ,CAAC;YACpC,WAAW,CAAC,QAAQ,IAAI,EAAE,CAAC,QAAQ,CAAC;YACpC,WAAW,CAAC,YAAY,IAAI,EAAE,CAAC,YAAY,CAAC;QAChD,CAAC;QAED,WAAW,CAAC;YACR,IAAI,EAAE,OAAO;YACb,QAAQ;YACR,WAAW;SACd,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC,CAAC;IACvB,CAAC,EAAE,KAAK,CAAC,CAAC;AACd,CAAC"}
|
@@ -1,44 +0,0 @@
|
|
1
|
-
import { RuntimeWorker } from "./runtime/runtime-worker";
|
2
|
-
|
3
|
-
export interface PluginStats {
|
4
|
-
type: 'stats',
|
5
|
-
cpuUsage: NodeJS.CpuUsage;
|
6
|
-
memoryUsage: NodeJS.MemoryUsage;
|
7
|
-
}
|
8
|
-
|
9
|
-
export function startStatsUpdater(allMemoryStats: Map<RuntimeWorker, NodeJS.MemoryUsage>, updateStats: (stats: PluginStats) => Promise<void>) {
|
10
|
-
if (!updateStats)
|
11
|
-
console.warn('wtf')
|
12
|
-
setInterval(() => {
|
13
|
-
let cpuUsage: NodeJS.CpuUsage;
|
14
|
-
let memoryUsage: NodeJS.MemoryUsage;
|
15
|
-
if (process.cpuUsage)
|
16
|
-
cpuUsage = process.cpuUsage();
|
17
|
-
|
18
|
-
allMemoryStats.set(undefined, process.memoryUsage());
|
19
|
-
|
20
|
-
memoryUsage = {
|
21
|
-
rss: 0,
|
22
|
-
heapTotal: 0,
|
23
|
-
heapUsed: 0,
|
24
|
-
external: 0,
|
25
|
-
arrayBuffers: 0,
|
26
|
-
}
|
27
|
-
|
28
|
-
for (const mu of allMemoryStats.values()) {
|
29
|
-
if (!mu)
|
30
|
-
continue;
|
31
|
-
memoryUsage.rss += mu.rss;
|
32
|
-
memoryUsage.heapTotal += mu.heapTotal;
|
33
|
-
memoryUsage.heapUsed += mu.heapUsed;
|
34
|
-
memoryUsage.external += mu.external;
|
35
|
-
memoryUsage.arrayBuffers += mu.arrayBuffers;
|
36
|
-
}
|
37
|
-
|
38
|
-
updateStats({
|
39
|
-
type: 'stats',
|
40
|
-
cpuUsage,
|
41
|
-
memoryUsage,
|
42
|
-
}).catch(() => {});
|
43
|
-
}, 10000);
|
44
|
-
}
|