dcp-worker 3.2.30-1 → 3.2.30-3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/bin/dcp-worker +98 -61
- package/docs/CODEOWNERS +2 -0
- package/lib/blessed-components/index.js +1 -0
- package/lib/blessed-components/log.js +1 -0
- package/lib/check-scheduler-version.js +1 -0
- package/lib/dashboard-tui.js +184 -0
- package/lib/default-ui-events.js +187 -0
- package/lib/pidfile.js +1 -1
- package/lib/remote-console.js +10 -2
- package/lib/startWorkerLogger.js +93 -63
- package/lib/worker-loggers/console.js +24 -108
- package/lib/worker-loggers/dashboard.js +32 -173
- package/lib/worker-loggers/event-log.js +28 -60
- package/lib/worker-loggers/logfile.js +57 -83
- package/lib/worker-loggers/syslog.js +41 -63
- package/package.json +4 -3
- package/lib/worker-loggers/common-types.js +0 -24
package/bin/dcp-worker
CHANGED
|
@@ -8,21 +8,16 @@
|
|
|
8
8
|
*/
|
|
9
9
|
'use strict';
|
|
10
10
|
|
|
11
|
+
var worker;
|
|
12
|
+
|
|
11
13
|
const process = require('process');
|
|
12
14
|
const fs = require('fs');
|
|
15
|
+
const path = require('path');
|
|
13
16
|
const crypto = require('crypto');
|
|
14
17
|
const chalk = require('chalk');
|
|
18
|
+
const telnetd = require('../lib/remote-console');
|
|
15
19
|
|
|
16
20
|
const configName = process.env.DCP_CONFIG || '../etc/dcp-worker-config';
|
|
17
|
-
var worker, dcpConfig, debugging;
|
|
18
|
-
|
|
19
|
-
const debug = (...args) => {
|
|
20
|
-
if (!debugging)
|
|
21
|
-
debugging = require('dcp/internal/debugging').scope('dcp-worker');
|
|
22
|
-
if (debugging())
|
|
23
|
-
console.debug('dcp-worker:', ...args);
|
|
24
|
-
};
|
|
25
|
-
|
|
26
21
|
const EXIT_UNHANDLED = 5;
|
|
27
22
|
|
|
28
23
|
/* Setup the telnet REPL up early to ensure early-failure log messages are captured */
|
|
@@ -34,11 +29,11 @@ const replHelpers = {
|
|
|
34
29
|
},
|
|
35
30
|
commands: {
|
|
36
31
|
report: printReport,
|
|
37
|
-
kill:
|
|
32
|
+
kill: processExit,
|
|
38
33
|
die: () => worker && worker.stop()
|
|
39
34
|
},
|
|
40
35
|
};
|
|
41
|
-
|
|
36
|
+
telnetd.init(replHelpers);
|
|
42
37
|
|
|
43
38
|
/* Initialize dcp-client with local config defaults and run the main function. DCP_CONFIG_COOKIE becomes dcpConfig.cookie. */
|
|
44
39
|
process.env.DCP_CONFIG_COOKIE = (Math.random().toString(16)).slice(2) + '-' + process.pid + '-' + Date.now();
|
|
@@ -48,7 +43,6 @@ function parseCliArgs()
|
|
|
48
43
|
{
|
|
49
44
|
var defaultPidFileName;
|
|
50
45
|
|
|
51
|
-
dcpConfig = require('dcp/dcp-config');
|
|
52
46
|
defaultPidFileName = require('../lib/pidfile').getDefaultPidFileName(dcpConfig.worker.pidfile);
|
|
53
47
|
|
|
54
48
|
const cliArgs = require('dcp/cli')
|
|
@@ -98,18 +92,21 @@ function parseCliArgs()
|
|
|
98
92
|
},
|
|
99
93
|
priorityOnly: {
|
|
100
94
|
alias: 'P',
|
|
95
|
+
hidden: true,
|
|
101
96
|
describe: 'Set the priority mode [deprecated]',
|
|
102
97
|
type: 'boolean',
|
|
103
98
|
default: false
|
|
104
99
|
},
|
|
105
100
|
'job-id': {
|
|
106
101
|
alias: 'j',
|
|
102
|
+
hidden: true,
|
|
107
103
|
describe: 'Restrict worker to a specific job (use N times for N jobs)',
|
|
108
104
|
type: 'array',
|
|
109
105
|
},
|
|
110
106
|
|
|
111
107
|
join: {
|
|
112
108
|
alias: 'g',
|
|
109
|
+
hidden: true,
|
|
113
110
|
describe: 'Join compute group; the format is "joinKey,joinSecret" or "joinKey,eh1-joinHash"',
|
|
114
111
|
type: 'array'
|
|
115
112
|
},
|
|
@@ -120,22 +117,27 @@ function parseCliArgs()
|
|
|
120
117
|
|
|
121
118
|
leavePublicGroup: {
|
|
122
119
|
type: 'boolean',
|
|
120
|
+
hidden: true,
|
|
123
121
|
describe: 'Do not fetch slices from public compute group.',
|
|
124
122
|
default: undefined,
|
|
125
123
|
},
|
|
126
124
|
|
|
127
125
|
publicGroupFallback: {
|
|
126
|
+
hidden: true,
|
|
128
127
|
describe: 'If set, worker will prefer private groups but fall back on the public group if no preferred work is available',
|
|
129
128
|
type: 'boolean',
|
|
130
|
-
default: undefined,
|
|
129
|
+
default: 'undefined',
|
|
130
|
+
defaultDescription: undefined,
|
|
131
131
|
},
|
|
132
132
|
|
|
133
133
|
identityKey: {
|
|
134
|
+
hidden: true,
|
|
134
135
|
describe: 'Identity key, in hex format',
|
|
135
136
|
type: 'string',
|
|
136
137
|
group: 'Identity options',
|
|
137
138
|
},
|
|
138
139
|
identityKeystore: {
|
|
140
|
+
hidden: true,
|
|
139
141
|
describe: 'Identity keystore, in json format',
|
|
140
142
|
type: 'string',
|
|
141
143
|
group: 'Identity options',
|
|
@@ -147,30 +149,40 @@ function parseCliArgs()
|
|
|
147
149
|
group: 'Output options',
|
|
148
150
|
},
|
|
149
151
|
eventDebug: {
|
|
150
|
-
|
|
152
|
+
hidden: true,
|
|
151
153
|
describe: 'If set, dump all sandbox and worker events',
|
|
152
154
|
},
|
|
153
155
|
|
|
154
156
|
logfile: {
|
|
155
|
-
describe: 'Path to log file
|
|
157
|
+
describe: 'Path to log file',
|
|
156
158
|
type: 'string',
|
|
157
159
|
group: 'Log File output options',
|
|
160
|
+
default: path.resolve('../log/dcp-worker.log'),
|
|
158
161
|
},
|
|
159
162
|
syslogAddress: {
|
|
160
|
-
describe: 'Address of
|
|
163
|
+
describe: 'Address of syslog server',
|
|
161
164
|
type: 'string',
|
|
162
165
|
group: 'Syslog output options',
|
|
166
|
+
default: 'loghost',
|
|
167
|
+
},
|
|
168
|
+
syslogFacility: {
|
|
169
|
+
describe: 'Name of syslog facility',
|
|
170
|
+
type: 'string',
|
|
171
|
+
group: 'Syslog output options',
|
|
172
|
+
default: 'local7',
|
|
163
173
|
},
|
|
164
174
|
syslogTransport: {
|
|
165
|
-
describe: 'Transport to connect to
|
|
175
|
+
describe: 'Transport to connect to use for syslog',
|
|
166
176
|
type: 'string',
|
|
167
|
-
choices: ['udp','tcp'],
|
|
177
|
+
choices: ['udp','tcp','unix','tls'],
|
|
168
178
|
group: 'Syslog output options',
|
|
179
|
+
default: 'udp',
|
|
169
180
|
},
|
|
170
181
|
syslogPort: {
|
|
171
|
-
describe: 'UDP/TCP port
|
|
182
|
+
describe: 'UDP/TCP port to use for syslog',
|
|
172
183
|
type: 'number',
|
|
173
184
|
group: 'Syslog output options',
|
|
185
|
+
default: 514,
|
|
174
186
|
},
|
|
175
187
|
|
|
176
188
|
allowedOrigins: {
|
|
@@ -202,8 +214,8 @@ function parseCliArgs()
|
|
|
202
214
|
|
|
203
215
|
if (cliArgs.dumpConfig)
|
|
204
216
|
{
|
|
205
|
-
console.
|
|
206
|
-
|
|
217
|
+
console.log(JSON.stringify(require('dcp/dcp-config'), null, 2));
|
|
218
|
+
processExit(0);
|
|
207
219
|
}
|
|
208
220
|
|
|
209
221
|
return cliArgs;
|
|
@@ -228,6 +240,20 @@ function addConfig(target, ...objs)
|
|
|
228
240
|
Object.assign(target, tmp);
|
|
229
241
|
}
|
|
230
242
|
|
|
243
|
+
/**
|
|
244
|
+
* Replacement for process.exit() that tries to increase the probability
|
|
245
|
+
* that remote log messages will make it out over the network.
|
|
246
|
+
*/
|
|
247
|
+
function processExit()
|
|
248
|
+
{
|
|
249
|
+
logClosing('debug', 'Exit Code:', process.exitCode || 0);
|
|
250
|
+
if (console.close)
|
|
251
|
+
console.close();
|
|
252
|
+
setImmediate(() => {
|
|
253
|
+
process.exit.apply(null, arguments);
|
|
254
|
+
});
|
|
255
|
+
}
|
|
256
|
+
|
|
231
257
|
/**
|
|
232
258
|
* Main program entry point. Assumes DCP client is already initialized and console logging is ready.
|
|
233
259
|
*/
|
|
@@ -235,21 +261,22 @@ async function main()
|
|
|
235
261
|
{
|
|
236
262
|
const wallet = require('dcp/wallet');
|
|
237
263
|
const DCPWorker = require('dcp/worker').Worker;
|
|
238
|
-
const { startWorkerLogger } = require('../lib/startWorkerLogger');
|
|
239
264
|
const cliArgs = parseCliArgs();
|
|
240
265
|
const sawOptions = {
|
|
241
266
|
hostname: cliArgs.hostname,
|
|
242
267
|
port: cliArgs.port
|
|
243
268
|
};
|
|
244
269
|
|
|
245
|
-
|
|
270
|
+
telnetd.setMainEval(function mainEval() { return eval(arguments[0]) }); // eslint-disable-line no-eval
|
|
271
|
+
require('../lib/startWorkerLogger').init(cliArgs); /* Start remote logger as early as possible */
|
|
272
|
+
verifyDefaultConfigIntegrity(); /* Bail before TUI & as early as possible if bad conf */
|
|
246
273
|
|
|
247
274
|
process.on('SIGINT', handleSigDeath);
|
|
248
275
|
process.on('SIGTERM', handleSigDeath);
|
|
249
276
|
process.on('SIGQUIT', handleSigDeath);
|
|
250
277
|
process.on('unhandledRejection', handleUnhandled);
|
|
251
278
|
process.on('uncaughtException', handleUnhandled);
|
|
252
|
-
|
|
279
|
+
|
|
253
280
|
let paymentAddress = false
|
|
254
281
|
|| cliArgs.paymentAddress
|
|
255
282
|
|| dcpConfig.worker.paymentAddress
|
|
@@ -285,6 +312,7 @@ async function main()
|
|
|
285
312
|
const dcpWorkerOptions = dcpConfig.worker;
|
|
286
313
|
const forceOptions = {
|
|
287
314
|
paymentAddress,
|
|
315
|
+
maxWorkingSandboxes: cliArgs.cores,
|
|
288
316
|
};
|
|
289
317
|
const defaultOptions = {
|
|
290
318
|
sandboxOptions: {
|
|
@@ -339,26 +367,26 @@ async function main()
|
|
|
339
367
|
dcpWorkerOptions.watchdogInterval = cliArgs.watchdogInterval;
|
|
340
368
|
|
|
341
369
|
worker = new DCPWorker(identityKeystore, dcpWorkerOptions);
|
|
342
|
-
worker.on('error',
|
|
343
|
-
worker.on('warning', (...payload) => console.warn(...payload));
|
|
370
|
+
worker.on('error', (...payload) => console.error(...payload));
|
|
371
|
+
worker.on('warning', (...payload) => console.warn (...payload));
|
|
372
|
+
worker.on('stop', () => { console.log('Worker is stopping') });
|
|
373
|
+
worker.on('end', () => { logClosing('log', 'Worker has stopped') });
|
|
374
|
+
require('../lib/default-ui-events').hook(worker, cliArgs);
|
|
344
375
|
|
|
376
|
+
if (cliArgs.outputMode === 'dashboard')
|
|
377
|
+
require('../lib/dashboard-tui').init(worker, cliArgs);
|
|
378
|
+
|
|
345
379
|
/* Let incorrect event-loop references keep us alive when linked with a debug library, but
|
|
346
380
|
* exit quickly/accurately for production code even when the library isn't perfect.
|
|
347
381
|
*/
|
|
348
382
|
if (require('dcp/build').config.build !== 'debug')
|
|
349
|
-
worker.on('end',
|
|
383
|
+
worker.on('end', processExit);
|
|
350
384
|
else
|
|
351
|
-
worker.on('end', () => setTimeout(
|
|
385
|
+
worker.on('end', () => setTimeout(processExit, getCleanupTimeoutMs()).unref());
|
|
352
386
|
|
|
353
387
|
if (cliArgs.eventDebug)
|
|
354
388
|
worker.enableDebugEvents = true;
|
|
355
389
|
|
|
356
|
-
worker.on('stop', () => { console.log('Worker is stopping') });
|
|
357
|
-
worker.on('end', () => { logClosing('log', 'Worker has stopped') });
|
|
358
|
-
startWorkerLogger(worker, cliArgs);
|
|
359
|
-
|
|
360
|
-
require('../lib/remote-console').setMainEval(function mainEval() { return eval(arguments[0]) });
|
|
361
|
-
|
|
362
390
|
if (dcpWorkerOptions.publicGroupFallback)
|
|
363
391
|
{
|
|
364
392
|
if (dcpWorkerOptions.leavePublicGroup)
|
|
@@ -380,12 +408,12 @@ async function main()
|
|
|
380
408
|
return;
|
|
381
409
|
}
|
|
382
410
|
|
|
383
|
-
if (typeof ev === 'string') /* <= June 2023 Worker events: remove ~ Sep 2023 /wg */
|
|
411
|
+
if (typeof ev === 'number' || typeof ev === 'string') /* <= June 2023 Worker events: remove ~ Sep 2023 /wg */
|
|
384
412
|
slicesFetched = ev;
|
|
385
413
|
else
|
|
386
414
|
{
|
|
387
415
|
const task = ev;
|
|
388
|
-
slicesFetched = task.slices
|
|
416
|
+
slicesFetched = task.slices.length;
|
|
389
417
|
}
|
|
390
418
|
|
|
391
419
|
dcpWorkerOptions.leavePublicGroup = Boolean(slicesFetched > 0);
|
|
@@ -415,21 +443,23 @@ async function main()
|
|
|
415
443
|
|
|
416
444
|
if (dcpWorkerOptions.jobAddresses?.length > 0)
|
|
417
445
|
introBanner += ` * Processing only ${qty(dcpWorkerOptions.jobAddresses, 'job')} ` + dcpWorkerOptions.jobAddresses.join(', ') + '\n';
|
|
418
|
-
if (dcpWorkerOptions.computeGroups
|
|
446
|
+
if (dcpWorkerOptions.computeGroups && Object.keys(dcpWorkerOptions.computeGroups).length > 0)
|
|
419
447
|
introBanner += ` . Joining compute ${qty(dcpWorkerOptions.computeGroups, 'group')} ` + dcpWorkerOptions.computeGroups.map(el => el.joinKey).join(', ') + '\n';
|
|
420
448
|
if (dcpWorkerOptions.publicGroupFallback)
|
|
421
449
|
introBanner += ' . Falling back on public group when preferred groups have no work' + '\n';
|
|
422
450
|
if (dcpWorkerOptions.leavePublicGroup)
|
|
423
451
|
introBanner += ' . Leaving the public compute group' + '\n';
|
|
424
452
|
if (dcpWorkerOptions.cores)
|
|
425
|
-
introBanner += ` .
|
|
453
|
+
introBanner += ` . Configured Cores: ${dcpWorkerOptions.cores.cpu},${dcpWorkerOptions.cores.gpu}\n`
|
|
426
454
|
else
|
|
427
455
|
introBanner += ` . Target core density: ${JSON.stringify(dcpWorkerOptions.defaultCoreDensity)}\n`;
|
|
428
456
|
if (cliArgs.verbose)
|
|
429
457
|
introBanner += ` + Verbosity level: ${cliArgs.verbose}` + '\n';
|
|
430
458
|
if (cliArgs.eventDebug)
|
|
431
459
|
introBanner += ' + Event debug on' + '\n';
|
|
432
|
-
|
|
460
|
+
if (telnetd.hasOwnProperty('port'))
|
|
461
|
+
introBanner += ` ! telnetd listening on port ${telnetd.port}\n`;
|
|
462
|
+
|
|
433
463
|
introBanner += ' . Supervisor version: ' + worker.supervisorVersion;
|
|
434
464
|
introBanner += ' . Output mode: ' + cliArgs.outputMode + '\n';
|
|
435
465
|
introBanner += ' * Ready' + '\n';
|
|
@@ -442,7 +472,7 @@ async function main()
|
|
|
442
472
|
if (cliArgs.outputMode !== 'dashboard')
|
|
443
473
|
setInterval(printReport, parseFloat(cliArgs.reportInterval) * 1000).unref();
|
|
444
474
|
else
|
|
445
|
-
console.
|
|
475
|
+
console.warn('Ignoring --reportInterval in dashboard output mode');
|
|
446
476
|
}
|
|
447
477
|
|
|
448
478
|
/* Start the worker. Normal process exit happens by virtue of the worker<end> event */
|
|
@@ -481,9 +511,9 @@ function processCoresAndDensity (dcpWorkerOptions, cliArgs)
|
|
|
481
511
|
parseArg('cores');
|
|
482
512
|
|
|
483
513
|
if (dcpWorkerOptions.cores)
|
|
484
|
-
debug('cores =
|
|
514
|
+
debugging() && console.debug('dcp-worker: cores =', dcpWorkerOptions.cores);
|
|
485
515
|
else
|
|
486
|
-
debug('core density =
|
|
516
|
+
debugging() && console.debug('dcp-worker: core density =', dcpWorkerOptions.defaultCoreDensity);
|
|
487
517
|
}
|
|
488
518
|
|
|
489
519
|
/**
|
|
@@ -494,9 +524,7 @@ function logClosing(facility, ...message)
|
|
|
494
524
|
{
|
|
495
525
|
var screen = require('../lib/worker-loggers/dashboard').screen;
|
|
496
526
|
|
|
497
|
-
if (
|
|
498
|
-
console[facility](message);
|
|
499
|
-
else
|
|
527
|
+
if (screen)
|
|
500
528
|
{
|
|
501
529
|
/* Turn off fullscreen TUI and resume "normal" console logging.
|
|
502
530
|
* FUTURE: dashboard API should know how to unregister its hook so that we don't have to clobber
|
|
@@ -505,10 +533,11 @@ function logClosing(facility, ...message)
|
|
|
505
533
|
screen.log(...message);
|
|
506
534
|
screen.destroy();
|
|
507
535
|
screen = false;
|
|
508
|
-
console = new (require('console').Console)(process);
|
|
509
|
-
|
|
510
|
-
console[facility].call(null, ...message);
|
|
536
|
+
console = new (require('console').Console)(process); // eslint-disable-line no-global-assign
|
|
537
|
+
telnetd.reintercept();
|
|
511
538
|
}
|
|
539
|
+
|
|
540
|
+
console[facility](...message);
|
|
512
541
|
}
|
|
513
542
|
|
|
514
543
|
/**
|
|
@@ -517,7 +546,7 @@ function logClosing(facility, ...message)
|
|
|
517
546
|
* the worker must be restarted. This handler does its best to report the rejection and give the worker a few
|
|
518
547
|
* seconds in which to attempt to return slices to the scheduler before it gives up completely.
|
|
519
548
|
*/
|
|
520
|
-
|
|
549
|
+
function handleUnhandled(error)
|
|
521
550
|
{
|
|
522
551
|
var _worker = worker;
|
|
523
552
|
worker = false;
|
|
@@ -526,21 +555,21 @@ async function handleUnhandled(error)
|
|
|
526
555
|
|
|
527
556
|
try
|
|
528
557
|
{
|
|
529
|
-
logClosing(error);
|
|
530
|
-
} catch(e) {}
|
|
558
|
+
logClosing('error', error);
|
|
559
|
+
} catch(e) {} // eslint-disable-line no-empty
|
|
531
560
|
|
|
532
561
|
if (!_worker)
|
|
533
562
|
console.error('trapped unhandled error:', error)
|
|
534
563
|
else
|
|
535
564
|
{
|
|
536
565
|
console.error('trapped unhandled error -- stopping worker:', error);
|
|
537
|
-
_worker.on('end',
|
|
566
|
+
_worker.on('end', processExit);
|
|
538
567
|
_worker.stop();
|
|
539
568
|
}
|
|
540
569
|
|
|
541
570
|
setTimeout(() => {
|
|
542
571
|
logClosing('error', 'handleFatalError timeout - exiting now');
|
|
543
|
-
|
|
572
|
+
processExit();
|
|
544
573
|
}, getCleanupTimeoutMs()).unref();
|
|
545
574
|
|
|
546
575
|
try {
|
|
@@ -550,7 +579,7 @@ async function handleUnhandled(error)
|
|
|
550
579
|
fs.appendFileSync(process.env.DCP_WORKER_UNHANDLED_REJECTION_LOG,
|
|
551
580
|
`${Date.now()}: ${error.message}\n${error.stack}\n\n`);
|
|
552
581
|
}
|
|
553
|
-
} catch(e) {}
|
|
582
|
+
} catch(e) {} // eslint-disable-line no-empty
|
|
554
583
|
}
|
|
555
584
|
|
|
556
585
|
/** print the slice report via console.log */
|
|
@@ -647,14 +676,14 @@ function handleSigDeath(signalName, signal)
|
|
|
647
676
|
process.off(signalName, handleSigDeath);
|
|
648
677
|
|
|
649
678
|
if (!worker)
|
|
650
|
-
console.
|
|
679
|
+
console.warn(`trapped ${signalName}, signal ${signal}`);
|
|
651
680
|
else
|
|
652
681
|
{
|
|
653
|
-
console.
|
|
682
|
+
console.warn(`trapped ${signalName}, signal ${signal} -- stopping worker`);
|
|
654
683
|
worker.stop(signalName === 'SIGQUIT');
|
|
655
684
|
}
|
|
656
685
|
|
|
657
|
-
setTimeout(() =>
|
|
686
|
+
setTimeout(() => processExit(signal - 128), getCleanupTimeoutMs()).unref();
|
|
658
687
|
}
|
|
659
688
|
|
|
660
689
|
/**
|
|
@@ -674,7 +703,7 @@ function getCleanupTimeoutMs()
|
|
|
674
703
|
cleanupTimeout = defaultCT;
|
|
675
704
|
if (!getCleanupTimeoutMs.warned)
|
|
676
705
|
{
|
|
677
|
-
console.
|
|
706
|
+
console.error(`warning: dcpConfig.worker.cleanupTimeout is not a number (${dcpConfig.worker.cleanupTimeout})`);
|
|
678
707
|
getCleanupTimeoutMs.warned = true;
|
|
679
708
|
}
|
|
680
709
|
}
|
|
@@ -696,7 +725,7 @@ function verifyDefaultConfigIntegrity()
|
|
|
696
725
|
|
|
697
726
|
if (!fs.existsSync(md5sumPath))
|
|
698
727
|
{
|
|
699
|
-
console.
|
|
728
|
+
console.error(chalk.bold.red(` ! warning: ${md5sumPath} not found; cannot verify configuration integrity`));
|
|
700
729
|
require('dcp/utils').sleep(2);
|
|
701
730
|
}
|
|
702
731
|
else
|
|
@@ -718,7 +747,7 @@ function verifyDefaultConfigIntegrity()
|
|
|
718
747
|
console.warn(' - the Windows Registry');
|
|
719
748
|
|
|
720
749
|
if (require('dcp/build').config.build !== 'debug')
|
|
721
|
-
|
|
750
|
+
processExit(1);
|
|
722
751
|
|
|
723
752
|
console.log(chalk.bold.red.inverse("If this wasn't a debug build, the worker would exit now."));
|
|
724
753
|
require('dcp/utils').sleep(2);
|
|
@@ -728,10 +757,18 @@ function verifyDefaultConfigIntegrity()
|
|
|
728
757
|
if (dcpConfig.cookie !== process.env.DCP_CONFIG_COOKIE || !dcpConfig.cookie)
|
|
729
758
|
{
|
|
730
759
|
console.error(' ! DCP Worker default configuration was not loaded; exiting.');
|
|
731
|
-
|
|
760
|
+
processExit(1);
|
|
732
761
|
}
|
|
733
762
|
}
|
|
734
763
|
|
|
764
|
+
/* thunk - ensures global debugging() symbol always available even if called before dcp-client init */
|
|
765
|
+
function debugging()
|
|
766
|
+
{
|
|
767
|
+
require('dcp-client');
|
|
768
|
+
debugging = require('dcp/internal/debugging').scope('dcp-worker');
|
|
769
|
+
return debugging.apply(this, arguments);
|
|
770
|
+
}
|
|
771
|
+
|
|
735
772
|
/**
|
|
736
773
|
* Cast b to boolean such that 'false' becomes false, falsey things become false, and everything else
|
|
737
774
|
* becomes true.
|
package/docs/CODEOWNERS
CHANGED
|
@@ -18,9 +18,11 @@
|
|
|
18
18
|
/package-lock.json @eroosenmaallen
|
|
19
19
|
|
|
20
20
|
[Wes]
|
|
21
|
+
/npm-hooks/ @wesgarland
|
|
21
22
|
/README.md @wesgarland
|
|
22
23
|
/docs/ @wesgarland
|
|
23
24
|
/.eslintrc.json @wesgarland
|
|
25
|
+
/.npmrc @wesgarland
|
|
24
26
|
/.tidelift @wesgarland
|
|
25
27
|
/lib/ @wesgarland
|
|
26
28
|
/LICENSE.md @wesgarland
|
|
@@ -0,0 +1,184 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @file worker-loggers/dashboard.js
|
|
3
|
+
* @author Ryan Rossiter, ryan@kingsds.network
|
|
4
|
+
* @date April 2020
|
|
5
|
+
* @author Wes Garland, wes@distributive.network
|
|
6
|
+
* @date June 2023
|
|
7
|
+
*
|
|
8
|
+
* This module uses the blessed library to create a monitoring dashboard for the worker.
|
|
9
|
+
* A corresponding worker-logger, dashboard.js, knows how to log to this dashboard.
|
|
10
|
+
*/
|
|
11
|
+
'use strict';
|
|
12
|
+
|
|
13
|
+
const dcpConfig = require('dcp/dcp-config');
|
|
14
|
+
const chalk = require('chalk');
|
|
15
|
+
const blessed = require('blessed');
|
|
16
|
+
const contrib = require('blessed-contrib');
|
|
17
|
+
const components = require('./blessed-components');
|
|
18
|
+
|
|
19
|
+
const { replaceWorkerEvent, replaceSandboxEvent } = require('./default-ui-events');
|
|
20
|
+
|
|
21
|
+
const SLICE_FETCH_STATUS = {
|
|
22
|
+
IDLE: chalk.yellow('Idle'),
|
|
23
|
+
FETCHING: chalk.blue('Fetching Work...'),
|
|
24
|
+
WORKING: chalk.green('Working'),
|
|
25
|
+
NO_WORK: chalk.red('No Work Available'),
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
const usingDebugger = require('module')._cache.niim instanceof require('module').Module;
|
|
29
|
+
const screenConf = {
|
|
30
|
+
input: usingDebugger ? new (require('events').EventEmitter) : undefined,
|
|
31
|
+
output: usingDebugger ? new (require('events').EventEmitter) : undefined,
|
|
32
|
+
};
|
|
33
|
+
/**
|
|
34
|
+
*/
|
|
35
|
+
exports.init = function dashboard$$init(worker, options)
|
|
36
|
+
{
|
|
37
|
+
var sliceFetchStatus = SLICE_FETCH_STATUS.IDLE;
|
|
38
|
+
var totalDCCs = 0;
|
|
39
|
+
const screen = blessed.screen(screenConf);
|
|
40
|
+
const grid = new contrib.grid({rows: 3, cols: 5, screen});
|
|
41
|
+
const workerInfoPane = grid.set(2, 0, 1, 5, blessed.text);
|
|
42
|
+
|
|
43
|
+
const logPane = grid.set(0, 2, 2, 3, components.log, {
|
|
44
|
+
label: 'Worker Log',
|
|
45
|
+
scrollable: true,
|
|
46
|
+
alwaysScroll: true,
|
|
47
|
+
mouse: true,
|
|
48
|
+
scrollbar: {
|
|
49
|
+
bg: 'blue',
|
|
50
|
+
},
|
|
51
|
+
});
|
|
52
|
+
|
|
53
|
+
const sandboxPane = grid.set(0, 0, 2, 2, components.sandboxes, {
|
|
54
|
+
label: 'Sandboxes',
|
|
55
|
+
defaultProgressBars: Math.floor(worker.workerOptions.cores.cpu) || 1,
|
|
56
|
+
scrollable: true,
|
|
57
|
+
alwaysScroll: true,
|
|
58
|
+
mouse: true,
|
|
59
|
+
scrollbar: {
|
|
60
|
+
bg: 'blue',
|
|
61
|
+
},
|
|
62
|
+
});
|
|
63
|
+
|
|
64
|
+
delete exports.init; /* singleton */
|
|
65
|
+
|
|
66
|
+
if (!usingDebugger)
|
|
67
|
+
exports.logPane = logPane; /* now dashboard log can find the pane */
|
|
68
|
+
setInterval(() => screen.render(), 50).unref(); /* 50ms = 20 fps */
|
|
69
|
+
updateWorkerInfo();
|
|
70
|
+
screen.render();
|
|
71
|
+
|
|
72
|
+
/* Apply key bindings which mimic canonical input mode */
|
|
73
|
+
screen.key(['C-c'], () => raise('SIGINT'));
|
|
74
|
+
screen.key(['C-z'], () => raise('SIGTSTP'));
|
|
75
|
+
screen.key(['\u001c'], () => raise('SIGQUIT')); /* C-\ */
|
|
76
|
+
|
|
77
|
+
screen.key(['escape'], () => {
|
|
78
|
+
console.log('Stopping worker...');
|
|
79
|
+
worker.stop();
|
|
80
|
+
});
|
|
81
|
+
|
|
82
|
+
function updateWorkerInfo()
|
|
83
|
+
{
|
|
84
|
+
const workerOptions = worker.workerOptions;
|
|
85
|
+
|
|
86
|
+
workerInfoPane.setLabel(`Worker Status [${sliceFetchStatus}]`);
|
|
87
|
+
workerInfoPane.setContent([
|
|
88
|
+
chalk.green(` DCCs Earned: ${chalk.bold(totalDCCs.toFixed(3))}`),
|
|
89
|
+
'',
|
|
90
|
+
` Scheduler: ${chalk.yellow(dcpConfig.scheduler.location.href)}`,
|
|
91
|
+
` Bank: ${chalk.yellow(dcpConfig.bank.location.href)}`,
|
|
92
|
+
`Bank Account: ${chalk.yellow(worker.paymentAddress || 'Starting...')}`,
|
|
93
|
+
` Identity: ${chalk.yellow(worker.identityKeystore? worker.identityKeystore.address : 'Starting...')}`,
|
|
94
|
+
` Jobs: ${workerOptions.jobAddresses?.length ? workerOptions.jobAddresses.join(', ') : '<any>'}`,
|
|
95
|
+
` Priv Groups: ${Object.keys(workerOptions.computeGroups).length}`,
|
|
96
|
+
` Pub Group: ${workerOptions.leavePublicGroup ? 'no' : 'yes'}`,
|
|
97
|
+
].join('\n'));
|
|
98
|
+
}
|
|
99
|
+
|
|
100
|
+
/* Override default event behaviour to work better with the Dashboard. */
|
|
101
|
+
|
|
102
|
+
replaceSandboxEvent('start', function dashboard$$sliceStart(sandbox, sandboxData, ev) {
|
|
103
|
+
sandboxData.progressData = {
|
|
104
|
+
indeterminate: true,
|
|
105
|
+
progress: 0,
|
|
106
|
+
label: sandbox.public.name,
|
|
107
|
+
};
|
|
108
|
+
|
|
109
|
+
sandboxPane.data.push(sandboxData.progressData);
|
|
110
|
+
sandboxPane.update();
|
|
111
|
+
});
|
|
112
|
+
|
|
113
|
+
replaceSandboxEvent('sliceProgress', function dashboard$$sliceProgress(sandbox, sandboxData, ev) {
|
|
114
|
+
if (ev.indeterminate)
|
|
115
|
+
{
|
|
116
|
+
sandboxData.progressData.progress = 100;
|
|
117
|
+
setTimeout(() => {
|
|
118
|
+
if (sandboxData.progressData.indeterminate) {
|
|
119
|
+
sandboxData.progressData.progress = 0;
|
|
120
|
+
sandboxPane.update();
|
|
121
|
+
}
|
|
122
|
+
}, 500).unref();
|
|
123
|
+
}
|
|
124
|
+
else
|
|
125
|
+
{
|
|
126
|
+
sandboxData.progressData.progress = ev.progress;
|
|
127
|
+
sandboxData.progressData.indeterminate = false;
|
|
128
|
+
}
|
|
129
|
+
|
|
130
|
+
sandboxPane.update();
|
|
131
|
+
});
|
|
132
|
+
|
|
133
|
+
replaceSandboxEvent('sliceFinish', function dashboard$$sliceFinish(sandbox, sandboxData, ev) {
|
|
134
|
+
sandboxPane.data = sandboxPane.data.filter(d => d != sandboxData.progressData);
|
|
135
|
+
sandboxPane.update();
|
|
136
|
+
});
|
|
137
|
+
|
|
138
|
+
worker.on('payment', updateWorkerInfo);
|
|
139
|
+
|
|
140
|
+
replaceWorkerEvent('fetchStart', function dashboard$$fetchStart(ev) {
|
|
141
|
+
sliceFetchStatus = SLICE_FETCH_STATUS.FETCHING;
|
|
142
|
+
updateWorkerInfo();
|
|
143
|
+
});
|
|
144
|
+
|
|
145
|
+
replaceWorkerEvent('fetch', function dashboard$$fetch(ev) {
|
|
146
|
+
var slicesFetched;
|
|
147
|
+
|
|
148
|
+
if (ev instanceof Error)
|
|
149
|
+
{
|
|
150
|
+
console.error('Error fetching slices:', ev);
|
|
151
|
+
return;
|
|
152
|
+
}
|
|
153
|
+
|
|
154
|
+
if (typeof ev === 'number' || typeof ev === 'string') /* <= June 2023 Worker events: remove ~ Sep 2023 /wg */
|
|
155
|
+
slicesFetched = ev;
|
|
156
|
+
else
|
|
157
|
+
{
|
|
158
|
+
const task = ev;
|
|
159
|
+
slicesFetched = task.slices.length;
|
|
160
|
+
}
|
|
161
|
+
|
|
162
|
+
if (slicesFetched === 0 && sandboxPane.data.length === 0) {
|
|
163
|
+
sliceFetchStatus = SLICE_FETCH_STATUS.NO_WORK;
|
|
164
|
+
} else {
|
|
165
|
+
sliceFetchStatus = SLICE_FETCH_STATUS.WORKING;
|
|
166
|
+
}
|
|
167
|
+
|
|
168
|
+
updateWorkerInfo();
|
|
169
|
+
});
|
|
170
|
+
|
|
171
|
+
replaceWorkerEvent('fetchError', function dashabord$$fetchError() {
|
|
172
|
+
sliceFetchStatus = SLICE_FETCH_STATUS.NO_WORK;
|
|
173
|
+
updateWorkerInfo();
|
|
174
|
+
});
|
|
175
|
+
};
|
|
176
|
+
|
|
177
|
+
/**
|
|
178
|
+
* Send a signal to the caller
|
|
179
|
+
* @param {number|string} sig the signal to raise
|
|
180
|
+
*/
|
|
181
|
+
function raise(sig)
|
|
182
|
+
{
|
|
183
|
+
process.kill(process.pid, sig);
|
|
184
|
+
}
|