isotropic-cluster 0.4.0 → 0.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE.md +0 -0
- package/README.md +59 -18
- package/lib/cluster-primary.js +541 -2
- package/lib/cluster-worker.js +117 -2
- package/package.json +24 -19
package/LICENSE.md
CHANGED
|
File without changes
|
package/README.md
CHANGED
|
@@ -1,9 +1,9 @@
|
|
|
1
1
|
# isotropic-cluster
|
|
2
2
|
|
|
3
|
-
[
|
|
4
|
-
[
|
|
5
|
-
|
|
6
|
-
|
|
3
|
+
[npm version](https://www.npmjs.com/package/isotropic-cluster)
|
|
4
|
+
[License](https://github.com/ibi-group/isotropic-cluster/blob/main/LICENSE)
|
|
5
|
+
|
|
6
|
+
|
|
7
7
|
|
|
8
8
|
A reusable and extendable platform to manage local Node.js process clusters with a clean API for primary/worker communication and lifecycle management.
|
|
9
9
|
|
|
@@ -69,7 +69,7 @@ import _later from 'isotropic-later';
|
|
|
69
69
|
});
|
|
70
70
|
|
|
71
71
|
// Later, shutdown the cluster
|
|
72
|
-
|
|
72
|
+
_later(60000, () => {
|
|
73
73
|
primary.shutDown();
|
|
74
74
|
});
|
|
75
75
|
}
|
|
@@ -122,6 +122,7 @@ Node.js clusters consist of a primary process that manages multiple worker proce
|
|
|
122
122
|
- Handles worker failures and restarts
|
|
123
123
|
|
|
124
124
|
Workers are separate processes that:
|
|
125
|
+
|
|
125
126
|
- Handle specific tasks (like serving HTTP requests)
|
|
126
127
|
- Report their status to the primary
|
|
127
128
|
- Perform work assigned by the primary
|
|
@@ -205,13 +206,13 @@ import _http from 'node:http';
|
|
|
205
206
|
|
|
206
207
|
// Create HTTP server
|
|
207
208
|
const server = _http.createServer((request, response) => {
|
|
208
|
-
|
|
209
|
+
response.writeHead(200);
|
|
209
210
|
response.end(`Hello from worker ${_ClusterWorker.workerId}`);
|
|
210
211
|
});
|
|
211
212
|
|
|
212
213
|
// Start listening on port 3000
|
|
213
214
|
server.listen(3000, () => {
|
|
214
|
-
console.log(`Worker ${
|
|
215
|
+
console.log(`Worker ${_ClusterWorker.workerId} listening on port 3000`);
|
|
215
216
|
});
|
|
216
217
|
|
|
217
218
|
// Handle shutdown
|
|
@@ -304,7 +305,7 @@ import _ClusterPrimary from 'isotropic-cluster/lib/cluster-primary.js';
|
|
|
304
305
|
{
|
|
305
306
|
const primary = _ClusterPrimary(),
|
|
306
307
|
|
|
307
|
-
|
|
308
|
+
// Different types of tasks
|
|
308
309
|
tasksByType = {
|
|
309
310
|
compute: [{
|
|
310
311
|
data: '...',
|
|
@@ -326,7 +327,8 @@ import _ClusterPrimary from 'isotropic-cluster/lib/cluster-primary.js';
|
|
|
326
327
|
}, {
|
|
327
328
|
data: '...',
|
|
328
329
|
id: 'l2'
|
|
329
|
-
}]
|
|
330
|
+
}]
|
|
331
|
+
},
|
|
330
332
|
|
|
331
333
|
processQueue = queueType => {
|
|
332
334
|
const tasks = tasksByType[queueType];
|
|
@@ -354,7 +356,6 @@ import _ClusterPrimary from 'isotropic-cluster/lib/cluster-primary.js';
|
|
|
354
356
|
});
|
|
355
357
|
}
|
|
356
358
|
};
|
|
357
|
-
};
|
|
358
359
|
|
|
359
360
|
// Listen for worker readiness
|
|
360
361
|
primary.on('workerReady', () => {
|
|
@@ -373,8 +374,10 @@ import _ClusterPrimary from 'isotropic-cluster/lib/cluster-primary.js';
|
|
|
373
374
|
if (message.type === 'taskComplete') {
|
|
374
375
|
console.log(`Task ${message.taskId} completed with result:`, message.result);
|
|
375
376
|
|
|
376
|
-
// Process next task
|
|
377
|
-
processQueue();
|
|
377
|
+
// Process the next task from each queue
|
|
378
|
+
processQueue('compute');
|
|
379
|
+
processQueue('io');
|
|
380
|
+
processQueue('lightweight');
|
|
378
381
|
}
|
|
379
382
|
});
|
|
380
383
|
|
|
@@ -396,7 +399,7 @@ import _ClusterPrimary from 'isotropic-cluster/lib/cluster-primary.js';
|
|
|
396
399
|
import _make from 'isotropic-make';
|
|
397
400
|
|
|
398
401
|
// Extend ClusterPrimary with custom message handler methods
|
|
399
|
-
const _AdvancedPrimary = _make(_ClusterPrimary, {
|
|
402
|
+
const _AdvancedPrimary = _make('AdvancedPrimary', _ClusterPrimary, {
|
|
400
403
|
_eventWorkerMessage_log ({
|
|
401
404
|
data: {
|
|
402
405
|
message,
|
|
@@ -470,6 +473,8 @@ const primary = _ClusterPrimary(options);
|
|
|
470
473
|
```
|
|
471
474
|
|
|
472
475
|
Options:
|
|
476
|
+
|
|
477
|
+
- `restartBackoff`: Controls how dead workers are replaced. By default, replacements are paced by an [isotropic-backoff](https://www.npmjs.com/package/isotropic-backoff) instance with default levels. Pass a backoff configuration object to customize the levels, pass an existing backoff instance to share one, or pass `false` to disable the backoff and replace dead workers immediately and indefinitely.
|
|
473
478
|
- `workerArgs`: Array of arguments to pass to worker processes
|
|
474
479
|
- `workerScript`: Path to worker script (defaults to current script)
|
|
475
480
|
- `workerSilent`: Whether to suppress worker stdout/stderr
|
|
@@ -480,22 +485,27 @@ Options:
|
|
|
480
485
|
|
|
481
486
|
- **destroy()**: Clean up and destroy the primary instance
|
|
482
487
|
- **fork({ workerCount })**: Start new worker processes. `workerCount` is optional and defaults to `1`.
|
|
483
|
-
- **roundRobin({ tag })**: Select a worker using round-robin distribution. The `tag` is optional. It returns the worker object that has been least recently selected for the given tag.
|
|
488
|
+
- **roundRobin({ tag })**: Select a worker using round-robin distribution. The `tag` is optional. It returns the worker object that has been least recently selected for the given tag, or `undefined` if there are no workers.
|
|
484
489
|
- **send({ message, to })**: Send a message to one or more workers. `to` can be a worker object or a worker id. It can also be an array of either. Returns a promise.
|
|
485
490
|
- **shutDown()**: Gracefully shut down all workers
|
|
486
491
|
|
|
487
492
|
#### Properties
|
|
488
493
|
|
|
494
|
+
- **restartBackoff**: The backoff instance pacing worker replacements, or `undefined` when disabled
|
|
489
495
|
- **workerById**: Object mapping worker ids to worker objects
|
|
490
496
|
- **workers**: Array of all active worker objects
|
|
491
497
|
|
|
492
498
|
#### Events
|
|
493
499
|
|
|
500
|
+
- **addWorker**: When a ready worker is added to the pool
|
|
494
501
|
- **destroy**: When destroy begins
|
|
495
502
|
- **destroyComplete** When destroy completes
|
|
503
|
+
- **fork**: When `fork()` is called to start one or more workers
|
|
496
504
|
- **initialize**: When initialization begins
|
|
497
505
|
- **initializeComplete**: When initialization completes successfully
|
|
498
506
|
- **initializeError**: When initialization fails
|
|
507
|
+
- **removeWorker**: When a worker is removed from the pool
|
|
508
|
+
- **restartGiveUp**: When the restart backoff is exhausted and no workers remain; the cluster shuts down
|
|
499
509
|
- **shutDown**: When the cluster is shutting down
|
|
500
510
|
- **shutDownComplete**: When all workers have been shut down
|
|
501
511
|
- **workerDisconnect**: When a worker disconnects
|
|
@@ -516,11 +526,12 @@ const worker = _ClusterWorker(options);
|
|
|
516
526
|
```
|
|
517
527
|
|
|
518
528
|
Options:
|
|
529
|
+
|
|
519
530
|
- `initialize`: Whether to automatically initialize (defaults to true)
|
|
520
531
|
|
|
521
532
|
#### Methods
|
|
522
533
|
|
|
523
|
-
- **destroy()**: Clean up and destroy the worker instance
|
|
534
|
+
- **destroy({ timeout })**: Clean up and destroy the worker instance. The worker disconnects its IPC channel and, if it has not exited on its own within `timeout` milliseconds (default `6765`), is force-killed.
|
|
524
535
|
- **send({ message })**: Send a message to the primary process
|
|
525
536
|
|
|
526
537
|
#### Static Properties
|
|
@@ -548,7 +559,7 @@ import _make from 'isotropic-make';
|
|
|
548
559
|
import _mongoose from 'mongoose';
|
|
549
560
|
|
|
550
561
|
// Custom worker with database connection
|
|
551
|
-
const _DbWorker = _make(_ClusterWorker, {
|
|
562
|
+
const _DbWorker = _make('DbWorker', _ClusterWorker, {
|
|
552
563
|
get db () {
|
|
553
564
|
return this._db;
|
|
554
565
|
},
|
|
@@ -595,7 +606,36 @@ const _DbWorker = _make(_ClusterWorker, {
|
|
|
595
606
|
|
|
596
607
|
### Auto-Restart on Worker Failure
|
|
597
608
|
|
|
598
|
-
The `ClusterPrimary` automatically restarts workers that die unexpectedly
|
|
609
|
+
The `ClusterPrimary` automatically restarts workers that die unexpectedly, ensuring application resilience. Replacements are paced by a restart backoff: a worker that crashes repeatedly (for example, one that fails on startup) is replaced immediately once, then with escalating delays. Each worker that reaches the ready state resets the backoff. When the backoff is exhausted and other workers remain, the primary logs an error and continues with the remaining workers; when none remain, it publishes a `restartGiveUp` event and shuts down rather than looping forever.
|
|
610
|
+
|
|
611
|
+
```javascript
|
|
612
|
+
import _ClusterPrimary from 'isotropic-cluster/lib/cluster-primary.js';
|
|
613
|
+
|
|
614
|
+
{
|
|
615
|
+
// Customize the restart pacing: three immediate replacements, then five
|
|
616
|
+
// escalating delays from 500ms, then give up.
|
|
617
|
+
const primary = _ClusterPrimary({
|
|
618
|
+
restartBackoff: {
|
|
619
|
+
levels: [{
|
|
620
|
+
count: 3
|
|
621
|
+
}, {
|
|
622
|
+
count: 5,
|
|
623
|
+
delay: 500,
|
|
624
|
+
factor: 3,
|
|
625
|
+
maximumDelay: 30000
|
|
626
|
+
}]
|
|
627
|
+
}
|
|
628
|
+
});
|
|
629
|
+
|
|
630
|
+
primary.on('restartGiveUp', () => {
|
|
631
|
+
// The cluster could not keep any worker alive and is shutting down.
|
|
632
|
+
});
|
|
633
|
+
|
|
634
|
+
primary.fork({
|
|
635
|
+
workerCount: 4
|
|
636
|
+
});
|
|
637
|
+
}
|
|
638
|
+
```
|
|
599
639
|
|
|
600
640
|
### Health Monitoring
|
|
601
641
|
|
|
@@ -669,6 +709,7 @@ import _later from 'isotropic-later';
|
|
|
669
709
|
|
|
670
710
|
isotropic-cluster works seamlessly with other modules in the isotropic ecosystem:
|
|
671
711
|
|
|
712
|
+
- **isotropic-backoff**: Escalating delays for worker replacement
|
|
672
713
|
- **isotropic-error**: Nested error reporting
|
|
673
714
|
- **isotropic-initializable**: Parent-to-child initialization sequence
|
|
674
715
|
- **isotropic-logger**: Structured logging
|
|
@@ -681,4 +722,4 @@ Please refer to [CONTRIBUTING.md](https://github.com/ibi-group/isotropic-cluster
|
|
|
681
722
|
|
|
682
723
|
## Issues
|
|
683
724
|
|
|
684
|
-
If you encounter any issues, please file them at https://github.com/ibi-group/isotropic-cluster/issues
|
|
725
|
+
If you encounter any issues, please file them at [https://github.com/ibi-group/isotropic-cluster/issues](https://github.com/ibi-group/isotropic-cluster/issues)
|
package/lib/cluster-primary.js
CHANGED
|
@@ -1,2 +1,541 @@
|
|
|
1
|
-
import
|
|
2
|
-
//# sourceMappingURL=data:application/json;charset=utf-8;base64,eyJ2ZXJzaW9uIjozLCJuYW1lcyI6WyJfY2x1c3RlciIsIl9FcnJvciIsIl9Jbml0aWFsaXphYmxlIiwiX2xvZ2dlciIsIl9tYWtlIiwiX3BhdGgiLCJfcHJvY2VzcyIsIl9tb2R1bGVMb2dnZXIiLCJjaGlsZCIsIm1vZHVsZSIsImJhc2VuYW1lIiwiaW1wb3J0IiwibWV0YSIsImZpbGVuYW1lIiwiZXh0bmFtZSIsImZvcmsiLCJ3b3JrZXJDb3VudCIsIl9hY3RpdmUiLCJfcHVibGlzaCIsInJvdW5kUm9iaW4iLCJ0YWciLCJvbGRlc3RXb3JrZXIiLCJvbGRlc3RXb3JrZXJUaW1lIiwid29ya2VyVGltZUJ5V29ya2VyV2Vha01hcCIsIl93b3JrZXJUaW1lQnlXb3JrZXJXZWFrTWFwQnlUYWciLCJXZWFrTWFwIiwid29ya2VyIiwiX3dvcmtlcnMiLCJ3b3JrZXJUaW1lIiwiZ2V0Iiwic2V0IiwiRGF0ZSIsIm5vdyIsInNlbmQiLCJtZXNzYWdlIiwidG8iLCJBcnJheSIsImlzQXJyYXkiLCJQcm9taXNlIiwiYWxsIiwibWFwIiwicmVzb2x2ZSIsInJlamVjdCIsIl93b3JrZXJCeUlkIiwiZXJyb3IiLCJkZXRhaWxzIiwid29ya2VySWQiLCJpZCIsInNodXREb3duIiwid29ya2VyQnlJZCIsIndvcmtlcnMiLCJfYWRkV29ya2VyIiwiX2Rlc3Ryb3kiLCJhcmdzIiwiX29uY2VBZnRlciIsIk9iamVjdCIsImtleXMiLCJfY2x1c3Rlckxpc3RlbmVyQnlFdmVudE5hbWUiLCJmb3JFYWNoIiwiZXZlbnROYW1lIiwicmVtb3ZlTGlzdGVuZXIiLCJSZWZsZWN0IiwiYXBwbHkiLCJwcm90b3R5cGUiLCJfZXZlbnRBZGRXb3JrZXIiLCJkYXRhIiwidmFsdWVzIiwiX2V2ZW50Rm9yayIsImluZm8iLCJfZXZlbnRSZW1vdmVXb3JrZXIiLCJfZXZlbnRTaHV0RG93biIsImRpc2Nvbm5lY3QiLCJfZXZlbnRTaHV0RG93bkNvbXBsZXRlIiwiX2V2ZW50V29ya2VyRGlzY29ubmVjdCIsIl9yZW1vdmVXb3JrZXIiLCJfZXZlbnRXb3JrZXJFcnJvciIsImZhdGFsIiwiX2V2ZW50V29ya2VyRXhpdCIsImNvZGUiLCJzaWduYWwiLCJleGl0ZWRBZnRlckRpc2Nvbm5lY3QiLCJfZXZlbnRXb3JrZXJGb3JrIiwib24iLCJfZXZlbnRXb3JrZXJMaXN0ZW5pbmciLCJhZGRyZXNzIiwicG9ydCIsIl9ldmVudFdvcmtlck1lc3NhZ2UiLCJldmVudCIsInR5cGUiLCJtZXRob2QiLCJfZXZlbnRXb3JrZXJPbmxpbmUiLCJsaXN0ZW5lciIsIl9ldmVudFdvcmtlclJlYWR5IiwiX2luaXQiLCJleGl0IiwibGlzdGVuaW5nIiwiaGFuZGxlIiwib25saW5lIiwiY3JlYXRlIiwiX2luaXRpYWxpemUiLCJ3b3JrZXJBcmdzIiwid29ya2VyU2NyaXB0Iiwid29ya2VyU2lsZW50Iiwid29ya2VyU3RkaW8iLCJzZXR1cFByaW1hcnkiLCJhcmd2Iiwic2xpY2UiLCJleGVjIiwic2lsZW50Iiwic3RkaW8iLCJfcHVic3ViIiwiYWRkV29ya2VyIiwiY29tcGxldGVGdW5jdGlvbiIsInJlbW92ZVdvcmtlciIsImNvbXBsZXRlT25jZSIsInNodXREb3duQ29tcGxldGUiLCJwdWJsaXNoT25jZSIsIndvcmtlckRpc2Nvbm5lY3QiLCJ3b3JrZXJFcnJvciIsIndvcmtlckV4aXQiLCJ3b3JrZXJGb3JrIiwid29ya2VyTGlzdGVuaW5nIiwid29ya2VyTWVzc2FnZSIsIndvcmtlck9ubGluZSIsIndvcmtlclJlYWR5Il0sInNvdXJjZXMiOlsiLi4vanMvY2x1c3Rlci1wcmltYXJ5LmpzIl0sInNvdXJjZXNDb250ZW50IjpbImltcG9ydCBfY2x1c3RlciBmcm9tICdub2RlOmNsdXN0ZXInO1xuaW1wb3J0IF9FcnJvciBmcm9tICdpc290cm9waWMtZXJyb3InO1xuaW1wb3J0IF9Jbml0aWFsaXphYmxlIGZyb20gJ2lzb3Ryb3BpYy1pbml0aWFsaXphYmxlJztcbmltcG9ydCBfbG9nZ2VyIGZyb20gJ2lzb3Ryb3BpYy1sb2dnZXInO1xuaW1wb3J0IF9tYWtlIGZyb20gJ2lzb3Ryb3BpYy1tYWtlJztcbmltcG9ydCBfcGF0aCBmcm9tICdub2RlOnBhdGgnO1xuaW1wb3J0IF9wcm9jZXNzIGZyb20gJ25vZGU6cHJvY2Vzcyc7XG5cbmNvbnN0IF9tb2R1bGVMb2dnZXIgPSBfbG9nZ2VyLmNoaWxkKHtcbiAgICBtb2R1bGU6IF9wYXRoLmJhc2VuYW1lKGltcG9ydC5tZXRhLmZpbGVuYW1lLCBfcGF0aC5leHRuYW1lKGltcG9ydC5tZXRhLmZpbGVuYW1lKSlcbn0sIHRydWUpO1xuXG5leHBvcnQgZGVmYXVsdCBfbWFrZShfSW5pdGlhbGl6YWJsZSwge1xuICAgIGZvcmsgKHtcbiAgICAgICAgd29ya2VyQ291bnQgPSAxXG4gICAgfSA9IHt9KSB7XG4gICAgICAgIHJldHVybiB0aGlzLl9hY3RpdmUgP1xuICAgICAgICAgICAgdGhpcy5fcHVibGlzaCgnZm9yaycsIHtcbiAgICAgICAgICAgICAgICB3b3JrZXJDb3VudFxuICAgICAgICAgICAgfSkgOlxuICAgICAgICAgICAgdGhpcztcbiAgICB9LFxuICAgIHJvdW5kUm9iaW4gKHtcbiAgICAgICAgdGFnID0gJydcbiAgICB9ID0ge30pIHtcbiAgICAgICAgbGV0IG9sZGVzdFdvcmtlcixcbiAgICAgICAgICAgIG9sZGVzdFdvcmtlclRpbWUgPSBJbmZpbml0eSxcbiAgICAgICAgICAgIHdvcmtlclRpbWVCeVdvcmtlcldlYWtNYXAgPSB0aGlzLl93b3JrZXJUaW1lQnlXb3JrZXJXZWFrTWFwQnlUYWdbdGFnXTtcblxuICAgICAgICBpZiAoIXdvcmtlclRpbWVCeVdvcmtlcldlYWtNYXApIHtcbiAgICAgICAgICAgIHdvcmtlclRpbWVCeVdvcmtlcldlYWtNYXAgPSBuZXcgV2Vha01hcCgpO1xuICAgICAgICAgICAgdGhpcy5fd29ya2VyVGltZUJ5V29ya2VyV2Vha01hcEJ5VGFnW3RhZ10gPSB3b3JrZXJUaW1lQnlXb3JrZXJXZWFrTWFwO1xuICAgICAgICB9XG5cbiAgICAgICAgZm9yIChjb25zdCB3b3JrZXIgb2YgdGhpcy5fd29ya2Vycykge1xuICAgICAgICAgICAgY29uc3Qgd29ya2VyVGltZSA9IHdvcmtlclRpbWVCeVdvcmtlcldlYWtNYXAuZ2V0KHdvcmtlcik7XG5cbiAgICAgICAgICAgIGlmICh3b3JrZXJUaW1lKSB7XG4gICAgICAgICAgICAgICAgaWYgKHdvcmtlclRpbWUgPCBvbGRlc3RXb3JrZXJUaW1lKSB7XG4gICAgICAgICAgICAgICAgICAgIG9sZGVzdFdvcmtlciA9IHdvcmtlcjtcbiAgICAgICAgICAgICAgICAgICAgb2xkZXN0V29ya2VyVGltZSA9IHdvcmtlclRpbWU7XG4gICAgICAgICAgICAgICAgfVxuICAgICAgICAgICAgfSBlbHNlIHtcbiAgICAgICAgICAgICAgICBvbGRlc3RXb3JrZXIgPSB3b3JrZXI7XG4gICAgICAgICAgICAgICAgYnJlYWs7XG4gICAgICAgICAgICB9XG4gICAgICAgIH1cblxuICAgICAgICB3b3JrZXJUaW1lQnlXb3JrZXJXZWFrTWFwLnNldChvbGRlc3RXb3JrZXIsIERhdGUubm93KCkpO1xuICAgICAgICByZXR1cm4gb2xkZXN0V29ya2VyO1xuICAgIH0sXG4gICAgc2VuZCAoe1xuICAgICAgICBtZXNzYWdlLFxuICAgICAgICB0byA9IHRoaXMuX3dvcmtlcnNcbiAgICB9KSB7XG4gICAgICAgIGlmICghQXJyYXkuaXNBcnJheSh0bykpIHtcbiAgICAgICAgICAgIHRvID0gW1xuICAgICAgICAgICAgICAgIHRvXG4gICAgICAgICAgICBdO1xuICAgICAgICB9XG5cbiAgICAgICAgcmV0dXJuIFByb21pc2UuYWxsKHRvLm1hcCh0byA9PiBuZXcgUHJvbWlzZSgocmVzb2x2ZSwgcmVqZWN0KSA9PiB7XG4gICAgICAgICAgICBjb25zdCB3b3JrZXIgPSB0eXBlb2YgdG8gPT09ICdudW1iZXInID9cbiAgICAgICAgICAgICAgICB0aGlzLl93b3JrZXJCeUlkW3RvXSA6XG4gICAgICAgICAgICAgICAgdG87XG5cbiAgICAgICAgICAgIGlmICh3b3JrZXIgJiYgd29ya2VyLnNlbmQpIHtcbiAgICAgICAgICAgICAgICB3b3JrZXIuc2VuZChtZXNzYWdlLCBlcnJvciA9PiB7XG4gICAgICAgICAgICAgICAgICAgIGlmIChlcnJvcikge1xuICAgICAgICAgICAgICAgICAgICAgICAgcmVqZWN0KF9FcnJvcih7XG4gICAgICAgICAgICAgICAgICAgICAgICAgICAgZGV0YWlsczoge1xuICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICBtZXNzYWdlLFxuICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICB3b3JrZXJJZDogd29ya2VyLmlkXG4gICAgICAgICAgICAgICAgICAgICAgICAgICAgfSxcbiAgICAgICAgICAgICAgICAgICAgICAgICAgICBlcnJvcixcbiAgICAgICAgICAgICAgICAgICAgICAgICAgICBtZXNzYWdlOiAnRXJyb3Igc2VuZGluZyBtZXNzYWdlIHRvIHdvcmtlcidcbiAgICAgICAgICAgICAgICAgICAgICAgIH0pKTtcbiAgICAgICAgICAgICAgICAgICAgfSBlbHNlIHtcbiAgICAgICAgICAgICAgICAgICAgICAgIHJlc29sdmUoKTtcbiAgICAgICAgICAgICAgICAgICAgfVxuICAgICAgICAgICAgICAgIH0pO1xuICAgICAgICAgICAgfSBlbHNlIHtcbiAgICAgICAgICAgICAgICByZWplY3QoX0Vycm9yKHtcbiAgICAgICAgICAgICAgICAgICAgZGV0YWlsczoge1xuICAgICAgICAgICAgICAgICAgICAgICAgbWVzc2FnZSxcbiAgICAgICAgICAgICAgICAgICAgICAgIHRvXG4gICAgICAgICAgICAgICAgICAgIH0sXG4gICAgICAgICAgICAgICAgICAgIG1lc3NhZ2U6ICdJbnZhbGlkIG1lc3NhZ2UgZGVzdGluYXRpb24nXG4gICAgICAgICAgICAgICAgfSkpO1xuICAgICAgICAgICAgfVxuICAgICAgICB9KSkpO1xuICAgIH0sXG4gICAgc2h1dERvd24gKCkge1xuICAgICAgICByZXR1cm4gdGhpcy5fcHVibGlzaCgnc2h1dERvd24nKTtcbiAgICB9LFxuICAgIGdldCB3b3JrZXJCeUlkICgpIHtcbiAgICAgICAgcmV0dXJuIHRoaXMuX3dvcmtlckJ5SWQ7XG4gICAgfSxcbiAgICBnZXQgd29ya2VycyAoKSB7XG4gICAgICAgIHJldHVybiB0aGlzLl93b3JrZXJzO1xuICAgIH0sXG4gICAgX2FkZFdvcmtlciAoe1xuICAgICAgICB3b3JrZXJcbiAgICB9KSB7XG4gICAgICAgIHJldHVybiB0aGlzLl9wdWJsaXNoKCdhZGRXb3JrZXInLCB7XG4gICAgICAgICAgICB3b3JrZXJcbiAgICAgICAgfSk7XG4gICAgfSxcbiAgICBfZGVzdHJveSAoLi4uYXJncykge1xuICAgICAgICBpZiAodGhpcy5fYWN0aXZlKSB7XG4gICAgICAgICAgICB0aGlzLl9vbmNlQWZ0ZXIoJ3NodXREb3duQ29tcGxldGUnLCAnX2Rlc3Ryb3knKTtcbiAgICAgICAgICAgIHRoaXMuc2h1dERvd24oKTtcbiAgICAgICAgICAgIHJldHVybjtcbiAgICAgICAgfVxuXG4gICAgICAgIHRoaXMuX2FjdGl2ZSA9IHZvaWQgbnVsbDtcblxuICAgICAgICBPYmplY3Qua2V5cyh0aGlzLl9jbHVzdGVyTGlzdGVuZXJCeUV2ZW50TmFtZSkuZm9yRWFjaChldmVudE5hbWUgPT4ge1xuICAgICAgICAgICAgX2NsdXN0ZXIucmVtb3ZlTGlzdGVuZXIoZXZlbnROYW1lLCB0aGlzLl9jbHVzdGVyTGlzdGVuZXJCeUV2ZW50TmFtZVtldmVudE5hbWVdKTtcbiAgICAgICAgfSk7XG5cbiAgICAgICAgdGhpcy5fY2x1c3Rlckxpc3RlbmVyQnlFdmVudE5hbWUgPSB2b2lkIG51bGw7XG5cbiAgICAgICAgdGhpcy5fd29ya2VyVGltZUJ5V29ya2VyV2Vha01hcEJ5VGFnID0gdm9pZCBudWxsO1xuXG4gICAgICAgIFJlZmxlY3QuYXBwbHkoX0luaXRpYWxpemFibGUucHJvdG90eXBlLl9kZXN0cm95LCB0aGlzLCBhcmdzKTtcbiAgICB9LFxuICAgIF9ldmVudEFkZFdvcmtlciAoe1xuICAgICAgICBkYXRhOiB7XG4gICAgICAgICAgICB3b3JrZXJcbiAgICAgICAgfVxuICAgIH0pIHtcbiAgICAgICAgdGhpcy5fd29ya2VyQnlJZFt3b3JrZXIuaWRdID0gd29ya2VyO1xuICAgICAgICB0aGlzLl93b3JrZXJzID0gT2JqZWN0LnZhbHVlcyh0aGlzLl93b3JrZXJCeUlkKTtcbiAgICB9LFxuICAgIF9ldmVudEZvcmsgKHtcbiAgICAgICAgZGF0YToge1xuICAgICAgICAgICAgd29ya2VyQ291bnRcbiAgICAgICAgfVxuICAgIH0pIHtcbiAgICAgICAgd2hpbGUgKHdvcmtlckNvdW50ID4gMCkge1xuICAgICAgICAgICAgX21vZHVsZUxvZ2dlci5pbmZvKCdTdGFydGluZyBhIG5ldyB3b3JrZXInKTtcblxuICAgICAgICAgICAgX2NsdXN0ZXIuZm9yaygpO1xuICAgICAgICAgICAgd29ya2VyQ291bnQgLT0gMTtcbiAgICAgICAgfVxuICAgIH0sXG4gICAgX2V2ZW50UmVtb3ZlV29ya2VyICh7XG4gICAgICAgIGRhdGE6IHtcbiAgICAgICAgICAgIHdvcmtlclxuICAgICAgICB9XG4gICAgfSkge1xuICAgICAgICBkZWxldGUgdGhpcy5fd29ya2VyQnlJZFt3b3JrZXIuaWRdO1xuICAgICAgICB0aGlzLl93b3JrZXJzID0gT2JqZWN0LnZhbHVlcyh0aGlzLl93b3JrZXJCeUlkKTtcbiAgICB9LFxuICAgIF9ldmVudFNodXREb3duICgpIHtcbiAgICAgICAgX21vZHVsZUxvZ2dlci5pbmZvKCdTaHV0dGluZyBkb3duIHdvcmtlcnMnKTtcblxuICAgICAgICB0aGlzLl9hY3RpdmUgPSBmYWxzZTtcblxuICAgICAgICBfY2x1c3Rlci5kaXNjb25uZWN0KCgpID0+IHtcbiAgICAgICAgICAgIHRoaXMuX3B1Ymxpc2goJ3NodXREb3duQ29tcGxldGUnKTtcbiAgICAgICAgfSk7XG5cbiAgICAgICAgcmV0dXJuIHRoaXM7XG4gICAgfSxcbiAgICBfZXZlbnRTaHV0RG93bkNvbXBsZXRlICgpIHtcbiAgICAgICAgX21vZHVsZUxvZ2dlci5pbmZvKCdXb3JrZXIgc2h1dCBkb3duIGNvbXBsZXRlJyk7XG4gICAgfSxcbiAgICBfZXZlbnRXb3JrZXJEaXNjb25uZWN0ICh7XG4gICAgICAgIGRhdGE6IHtcbiAgICAgICAgICAgIHdvcmtlclxuICAgICAgICB9XG4gICAgfSkge1xuICAgICAgICBfbW9kdWxlTG9nZ2VyLmluZm8oe1xuICAgICAgICAgICAgd29ya2VySWQ6IHdvcmtlci5pZFxuICAgICAgICB9LCAnV29ya2VyIGRpc2Nvbm5lY3RlZCcpO1xuXG4gICAgICAgIHRoaXMuX3JlbW92ZVdvcmtlcih7XG4gICAgICAgICAgICB3b3JrZXJcbiAgICAgICAgfSk7XG4gICAgfSxcbiAgICBfZXZlbnRXb3JrZXJFcnJvciAoe1xuICAgICAgICBkYXRhOiB7XG4gICAgICAgICAgICBlcnJvcixcbiAgICAgICAgICAgIHdvcmtlclxuICAgICAgICB9XG4gICAgfSkge1xuICAgICAgICBfbW9kdWxlTG9nZ2VyLmZhdGFsKHtcbiAgICAgICAgICAgIGVycm9yOiBfRXJyb3Ioe1xuICAgICAgICAgICAgICAgIGRldGFpbHM6IHtcbiAgICAgICAgICAgICAgICAgICAgd29ya2VySWQ6IHdvcmtlci5pZFxuICAgICAgICAgICAgICAgIH0sXG4gICAgICAgICAgICAgICAgZXJyb3IsXG4gICAgICAgICAgICAgICAgbWVzc2FnZTogJ1dvcmtlciBlcnJvcidcbiAgICAgICAgICAgIH0pLFxuICAgICAgICAgICAgd29ya2VySWQ6IHdvcmtlci5pZFxuICAgICAgICB9LCAnV29ya2VyIGVycm9yJyk7XG5cbiAgICAgICAgdGhpcy5fcmVtb3ZlV29ya2VyKHtcbiAgICAgICAgICAgIHdvcmtlclxuICAgICAgICB9KTtcbiAgICB9LFxuICAgIF9ldmVudFdvcmtlckV4aXQgKHtcbiAgICAgICAgZGF0YToge1xuICAgICAgICAgICAgY29kZSxcbiAgICAgICAgICAgIHNpZ25hbCxcbiAgICAgICAgICAgIHdvcmtlclxuICAgICAgICB9XG4gICAgfSkge1xuICAgICAgICBpZiAod29ya2VyLmV4aXRlZEFmdGVyRGlzY29ubmVjdCkge1xuICAgICAgICAgICAgX21vZHVsZUxvZ2dlci5pbmZvKHtcbiAgICAgICAgICAgICAgICBjb2RlLFxuICAgICAgICAgICAgICAgIHNpZ25hbCxcbiAgICAgICAgICAgICAgICB3b3JrZXJJZDogd29ya2VyLmlkXG4gICAgICAgICAgICB9LCAnV29ya2VyIGV4aXRlZCB2b2x1bnRhcmlseScpO1xuICAgICAgICB9IGVsc2UgaWYgKGNvZGUpIHtcbiAgICAgICAgICAgIF9tb2R1bGVMb2dnZXIuZmF0YWwoe1xuICAgICAgICAgICAgICAgIGNvZGUsXG4gICAgICAgICAgICAgICAgc2lnbmFsLFxuICAgICAgICAgICAgICAgIHdvcmtlcklkOiB3b3JrZXIuaWRcbiAgICAgICAgICAgIH0sICdXb3JrZXIgZGllZCB1bmV4cGVjdGVkbHknKTtcbiAgICAgICAgfSBlbHNlIHtcbiAgICAgICAgICAgIF9tb2R1bGVMb2dnZXIuZXJyb3Ioe1xuICAgICAgICAgICAgICAgIGNvZGUsXG4gICAgICAgICAgICAgICAgc2lnbmFsLFxuICAgICAgICAgICAgICAgIHdvcmtlcklkOiB3b3JrZXIuaWRcbiAgICAgICAgICAgIH0sICdXb3JrZXIgZGllZCcpO1xuICAgICAgICB9XG5cbiAgICAgICAgdGhpcy5fcmVtb3ZlV29ya2VyKHtcbiAgICAgICAgICAgIHdvcmtlclxuICAgICAgICB9KTtcblxuICAgICAgICBpZiAodGhpcy5fYWN0aXZlKSB7XG4gICAgICAgICAgICBfbW9kdWxlTG9nZ2VyLmluZm8oJ1JlcGxhY2luZyBkZWFkIHdvcmtlcicpO1xuXG4gICAgICAgICAgICB0aGlzLmZvcmsoKTtcbiAgICAgICAgfVxuICAgIH0sXG4gICAgX2V2ZW50V29ya2VyRm9yayAoe1xuICAgICAgICBkYXRhOiB7XG4gICAgICAgICAgICB3b3JrZXJcbiAgICAgICAgfVxuICAgIH0pIHtcbiAgICAgICAgX21vZHVsZUxvZ2dlci5pbmZvKHtcbiAgICAgICAgICAgIHdvcmtlcklkOiB3b3JrZXIuaWRcbiAgICAgICAgfSwgJ1N0YXJ0aW5nIG5ldyB3b3JrZXInKTtcblxuICAgICAgICB3b3JrZXIub24oJ2Vycm9yJywgZXJyb3IgPT4ge1xuICAgICAgICAgICAgdGhpcy5fcHVibGlzaCgnd29ya2VyRXJyb3InLCB7XG4gICAgICAgICAgICAgICAgZXJyb3I6IF9FcnJvcih7XG4gICAgICAgICAgICAgICAgICAgIGRldGFpbHM6IHtcbiAgICAgICAgICAgICAgICAgICAgICAgIHdvcmtlclxuICAgICAgICAgICAgICAgICAgICB9LFxuICAgICAgICAgICAgICAgICAgICBlcnJvcixcbiAgICAgICAgICAgICAgICAgICAgbWVzc2FnZTogJ0Vycm9yIGluIHdvcmtlcidcbiAgICAgICAgICAgICAgICB9KSxcbiAgICAgICAgICAgICAgICB3b3JrZXJcbiAgICAgICAgICAgIH0pO1xuICAgICAgICB9KTtcbiAgICB9LFxuICAgIF9ldmVudFdvcmtlckxpc3RlbmluZyAoe1xuICAgICAgICBkYXRhOiB7XG4gICAgICAgICAgICBhZGRyZXNzLFxuICAgICAgICAgICAgd29ya2VyXG4gICAgICAgIH1cbiAgICB9KSB7XG4gICAgICAgIF9tb2R1bGVMb2dnZXIuaW5mbyh7XG4gICAgICAgICAgICBhZGRyZXNzOiBhZGRyZXNzLmFkZHJlc3MsXG4gICAgICAgICAgICBwb3J0OiBhZGRyZXNzLnBvcnQsXG4gICAgICAgICAgICB3b3JrZXJJZDogd29ya2VyLmlkXG4gICAgICAgIH0sICdXb3JrZXIgaXMgbGlzdGVuaW5nJyk7XG4gICAgfSxcbiAgICBfZXZlbnRXb3JrZXJNZXNzYWdlIChldmVudCkge1xuICAgICAgICBpZiAoZXZlbnQuZGF0YS5tZXNzYWdlICYmIGV2ZW50LmRhdGEubWVzc2FnZS50eXBlKSB7XG4gICAgICAgICAgICBjb25zdCBtZXRob2QgPSB0aGlzW2BfZXZlbnRXb3JrZXJNZXNzYWdlXyR7ZXZlbnQuZGF0YS5tZXNzYWdlLnR5cGV9YF07XG5cbiAgICAgICAgICAgIGlmICh0eXBlb2YgbWV0aG9kID09PSAnZnVuY3Rpb24nKSB7XG4gICAgICAgICAgICAgICAgUmVmbGVjdC5hcHBseShtZXRob2QsIHRoaXMsIFtcbiAgICAgICAgICAgICAgICAgICAgZXZlbnRcbiAgICAgICAgICAgICAgICBdKTtcbiAgICAgICAgICAgIH1cbiAgICAgICAgfVxuICAgIH0sXG4gICAgX2V2ZW50V29ya2VyT25saW5lICh7XG4gICAgICAgIGRhdGE6IHtcbiAgICAgICAgICAgIHdvcmtlclxuICAgICAgICB9XG4gICAgfSkge1xuICAgICAgICBfbW9kdWxlTG9nZ2VyLmluZm8oe1xuICAgICAgICAgICAgd29ya2VySWQ6IHdvcmtlci5pZFxuICAgICAgICB9LCAnV29ya2VyIGlzIG5vdyBvbmxpbmUnKTtcblxuICAgICAgICBjb25zdCBsaXN0ZW5lciA9IG1lc3NhZ2UgPT4ge1xuICAgICAgICAgICAgaWYgKG1lc3NhZ2UgPT09ICdyZWFkeScpIHtcbiAgICAgICAgICAgICAgICB3b3JrZXIucmVtb3ZlTGlzdGVuZXIoJ21lc3NhZ2UnLCBsaXN0ZW5lcik7XG5cbiAgICAgICAgICAgICAgICB0aGlzLl9wdWJsaXNoKCd3b3JrZXJSZWFkeScsIHtcbiAgICAgICAgICAgICAgICAgICAgd29ya2VyXG4gICAgICAgICAgICAgICAgfSk7XG4gICAgICAgICAgICB9XG4gICAgICAgIH07XG5cbiAgICAgICAgd29ya2VyLm9uKCdtZXNzYWdlJywgbGlzdGVuZXIpO1xuICAgIH0sXG4gICAgX2V2ZW50V29ya2VyUmVhZHkgKHtcbiAgICAgICAgZGF0YToge1xuICAgICAgICAgICAgd29ya2VyXG4gICAgICAgIH1cbiAgICB9KSB7XG4gICAgICAgIF9tb2R1bGVMb2dnZXIuaW5mbyh7XG4gICAgICAgICAgICB3b3JrZXJJZDogd29ya2VyLmlkXG4gICAgICAgIH0sICdXb3JrZXIgaXMgcmVhZHkgdG8gd29yaycpO1xuXG4gICAgICAgIHRoaXMuX2FkZFdvcmtlcih7XG4gICAgICAgICAgICB3b3JrZXJcbiAgICAgICAgfSk7XG4gICAgfSxcbiAgICBfaW5pdCAoLi4uYXJncykge1xuICAgICAgICB0aGlzLl9hY3RpdmUgPSB0cnVlO1xuXG4gICAgICAgIHRoaXMuX2NsdXN0ZXJMaXN0ZW5lckJ5RXZlbnROYW1lID0ge1xuICAgICAgICAgICAgZGlzY29ubmVjdDogd29ya2VyID0+IHtcbiAgICAgICAgICAgICAgICB0aGlzLl9wdWJsaXNoKCd3b3JrZXJEaXNjb25uZWN0Jywge1xuICAgICAgICAgICAgICAgICAgICB3b3JrZXJcbiAgICAgICAgICAgICAgICB9KTtcbiAgICAgICAgICAgIH0sXG4gICAgICAgICAgICBleGl0OiAod29ya2VyLCBjb2RlLCBzaWduYWwpID0+IHtcbiAgICAgICAgICAgICAgICB0aGlzLl9wdWJsaXNoKCd3b3JrZXJFeGl0Jywge1xuICAgICAgICAgICAgICAgICAgICBjb2RlLFxuICAgICAgICAgICAgICAgICAgICBzaWduYWwsXG4gICAgICAgICAgICAgICAgICAgIHdvcmtlclxuICAgICAgICAgICAgICAgIH0pO1xuICAgICAgICAgICAgfSxcbiAgICAgICAgICAgIGZvcms6IHdvcmtlciA9PiB7XG4gICAgICAgICAgICAgICAgdGhpcy5fcHVibGlzaCgnd29ya2VyRm9yaycsIHtcbiAgICAgICAgICAgICAgICAgICAgd29ya2VyXG4gICAgICAgICAgICAgICAgfSk7XG4gICAgICAgICAgICB9LFxuICAgICAgICAgICAgbGlzdGVuaW5nOiAod29ya2VyLCBhZGRyZXNzKSA9PiB7XG4gICAgICAgICAgICAgICAgdGhpcy5fcHVibGlzaCgnd29ya2VyTGlzdGVuaW5nJywge1xuICAgICAgICAgICAgICAgICAgICBhZGRyZXNzLFxuICAgICAgICAgICAgICAgICAgICB3b3JrZXJcbiAgICAgICAgICAgICAgICB9KTtcbiAgICAgICAgICAgIH0sXG4gICAgICAgICAgICBtZXNzYWdlOiAod29ya2VyLCBtZXNzYWdlLCBoYW5kbGUpID0+IHtcbiAgICAgICAgICAgICAgICB0aGlzLl9wdWJsaXNoKCd3b3JrZXJNZXNzYWdlJywge1xuICAgICAgICAgICAgICAgICAgICBoYW5kbGUsXG4gICAgICAgICAgICAgICAgICAgIG1lc3NhZ2UsXG4gICAgICAgICAgICAgICAgICAgIHdvcmtlclxuICAgICAgICAgICAgICAgIH0pO1xuICAgICAgICAgICAgfSxcbiAgICAgICAgICAgIG9ubGluZTogd29ya2VyID0+IHtcbiAgICAgICAgICAgICAgICB0aGlzLl9wdWJsaXNoKCd3b3JrZXJPbmxpbmUnLCB7XG4gICAgICAgICAgICAgICAgICAgIHdvcmtlclxuICAgICAgICAgICAgICAgIH0pO1xuICAgICAgICAgICAgfVxuICAgICAgICB9O1xuXG4gICAgICAgIHRoaXMuX3dvcmtlckJ5SWQgPSBPYmplY3QuY3JlYXRlKG51bGwpO1xuXG4gICAgICAgIHRoaXMuX3dvcmtlcnMgPSBbXTtcblxuICAgICAgICB0aGlzLl93b3JrZXJUaW1lQnlXb3JrZXJXZWFrTWFwQnlUYWcgPSBPYmplY3QuY3JlYXRlKG51bGwpO1xuXG4gICAgICAgIHJldHVybiBSZWZsZWN0LmFwcGx5KF9Jbml0aWFsaXphYmxlLnByb3RvdHlwZS5faW5pdCwgdGhpcywgYXJncyk7XG4gICAgfSxcbiAgICBfaW5pdGlhbGl6ZSAoLi4uYXJncykge1xuICAgICAgICBfbW9kdWxlTG9nZ2VyLmluZm8oJ0luaXRpYWxpemluZyBwcmltYXJ5Jyk7XG5cbiAgICAgICAgY29uc3QgW3tcbiAgICAgICAgICAgIHdvcmtlckFyZ3MsXG4gICAgICAgICAgICB3b3JrZXJTY3JpcHQsXG4gICAgICAgICAgICB3b3JrZXJTaWxlbnQsXG4gICAgICAgICAgICB3b3JrZXJTdGRpb1xuICAgICAgICB9ID0ge31dID0gYXJncztcblxuICAgICAgICBfY2x1c3Rlci5zZXR1cFByaW1hcnkoe1xuICAgICAgICAgICAgYXJnczogdHlwZW9mIHdvcmtlckFyZ3MgPT09ICd1bmRlZmluZWQnID9cbiAgICAgICAgICAgICAgICBfcHJvY2Vzcy5hcmd2LnNsaWNlKDIpIDpcbiAgICAgICAgICAgICAgICB3b3JrZXJBcmdzLFxuICAgICAgICAgICAgZXhlYzogdHlwZW9mIHdvcmtlclNjcmlwdCA9PT0gJ3VuZGVmaW5lZCcgP1xuICAgICAgICAgICAgICAgIF9wcm9jZXNzLmFyZ3ZbMV0gOlxuICAgICAgICAgICAgICAgIHdvcmtlclNjcmlwdCxcbiAgICAgICAgICAgIHNpbGVudDogdHlwZW9mIHdvcmtlclNpbGVudCA9PT0gJ3VuZGVmaW5lZCcgP1xuICAgICAgICAgICAgICAgIGZhbHNlIDpcbiAgICAgICAgICAgICAgICB3b3JrZXJTaWxlbnQsXG4gICAgICAgICAgICBzdGRpbzogd29ya2VyU3RkaW9cbiAgICAgICAgfSk7XG5cbiAgICAgICAgT2JqZWN0LmtleXModGhpcy5fY2x1c3Rlckxpc3RlbmVyQnlFdmVudE5hbWUpLmZvckVhY2goZXZlbnROYW1lID0+IHtcbiAgICAgICAgICAgIF9jbHVzdGVyLm9uKGV2ZW50TmFtZSwgdGhpcy5fY2x1c3Rlckxpc3RlbmVyQnlFdmVudE5hbWVbZXZlbnROYW1lXSk7XG4gICAgICAgIH0pO1xuICAgIH0sXG4gICAgX3JlbW92ZVdvcmtlciAoe1xuICAgICAgICB3b3JrZXJcbiAgICB9KSB7XG4gICAgICAgIGlmICh3b3JrZXIuaWQgaW4gdGhpcy5fd29ya2VyQnlJZCkge1xuICAgICAgICAgICAgdGhpcy5fcHVibGlzaCgncmVtb3ZlV29ya2VyJywge1xuICAgICAgICAgICAgICAgIHdvcmtlclxuICAgICAgICAgICAgfSk7XG4gICAgICAgIH1cblxuICAgICAgICByZXR1cm4gdGhpcztcbiAgICB9XG59LCB7XG4gICAgX3B1YnN1Yjoge1xuICAgICAgICBhZGRXb3JrZXI6IHtcbiAgICAgICAgICAgIGNvbXBsZXRlRnVuY3Rpb246ICdfZXZlbnRBZGRXb3JrZXInXG4gICAgICAgIH0sXG4gICAgICAgIGZvcms6IHtcbiAgICAgICAgICAgIGNvbXBsZXRlRnVuY3Rpb246ICdfZXZlbnRGb3JrJ1xuICAgICAgICB9LFxuICAgICAgICByZW1vdmVXb3JrZXI6IHtcbiAgICAgICAgICAgIGNvbXBsZXRlRnVuY3Rpb246ICdfZXZlbnRSZW1vdmVXb3JrZXInXG4gICAgICAgIH0sXG4gICAgICAgIHNodXREb3duOiB7XG4gICAgICAgICAgICBjb21wbGV0ZUZ1bmN0aW9uOiAnX2V2ZW50U2h1dERvd24nLFxuICAgICAgICAgICAgY29tcGxldGVPbmNlOiB0cnVlXG4gICAgICAgIH0sXG4gICAgICAgIHNodXREb3duQ29tcGxldGU6IHtcbiAgICAgICAgICAgIGNvbXBsZXRlRnVuY3Rpb246ICdfZXZlbnRTaHV0RG93bkNvbXBsZXRlJyxcbiAgICAgICAgICAgIHB1Ymxpc2hPbmNlOiB0cnVlXG4gICAgICAgIH0sXG4gICAgICAgIHdvcmtlckRpc2Nvbm5lY3Q6IHtcbiAgICAgICAgICAgIGNvbXBsZXRlRnVuY3Rpb246ICdfZXZlbnRXb3JrZXJEaXNjb25uZWN0J1xuICAgICAgICB9LFxuICAgICAgICB3b3JrZXJFcnJvcjoge1xuICAgICAgICAgICAgY29tcGxldGVGdW5jdGlvbjogJ19ldmVudFdvcmtlckVycm9yJ1xuICAgICAgICB9LFxuICAgICAgICB3b3JrZXJFeGl0OiB7XG4gICAgICAgICAgICBjb21wbGV0ZUZ1bmN0aW9uOiAnX2V2ZW50V29ya2VyRXhpdCdcbiAgICAgICAgfSxcbiAgICAgICAgd29ya2VyRm9yazoge1xuICAgICAgICAgICAgY29tcGxldGVGdW5jdGlvbjogJ19ldmVudFdvcmtlckZvcmsnXG4gICAgICAgIH0sXG4gICAgICAgIHdvcmtlckxpc3RlbmluZzoge1xuICAgICAgICAgICAgY29tcGxldGVGdW5jdGlvbjogJ19ldmVudFdvcmtlckxpc3RlbmluZydcbiAgICAgICAgfSxcbiAgICAgICAgd29ya2VyTWVzc2FnZToge1xuICAgICAgICAgICAgY29tcGxldGVGdW5jdGlvbjogJ19ldmVudFdvcmtlck1lc3NhZ2UnXG4gICAgICAgIH0sXG4gICAgICAgIHdvcmtlck9ubGluZToge1xuICAgICAgICAgICAgY29tcGxldGVGdW5jdGlvbjogJ19ldmVudFdvcmtlck9ubGluZSdcbiAgICAgICAgfSxcbiAgICAgICAgd29ya2VyUmVhZHk6IHtcbiAgICAgICAgICAgIGNvbXBsZXRlRnVuY3Rpb246ICdfZXZlbnRXb3JrZXJSZWFkeSdcbiAgICAgICAgfVxuICAgIH1cbn0pO1xuIl0sIm1hcHBpbmdzIjoiQUFBQSxNQUFPLENBQUFBLFFBQVEsS0FBTSxjQUFjLENBQ25DLE1BQU8sQ0FBQUMsTUFBTSxLQUFNLGlCQUFpQixDQUNwQyxNQUFPLENBQUFDLGNBQWMsS0FBTSx5QkFBeUIsQ0FDcEQsTUFBTyxDQUFBQyxPQUFPLEtBQU0sa0JBQWtCLENBQ3RDLE1BQU8sQ0FBQUMsS0FBSyxLQUFNLGdCQUFnQixDQUNsQyxNQUFPLENBQUFDLEtBQUssS0FBTSxXQUFXLENBQzdCLE1BQU8sQ0FBQUMsUUFBUSxLQUFNLGNBQWMsQ0FFbkMsS0FBTSxDQUFBQyxhQUFhLENBQUdKLE9BQU8sQ0FBQ0ssS0FBSyxDQUFDLENBQ2hDQyxNQUFNLENBQUVKLEtBQUssQ0FBQ0ssUUFBUSxDQUFDQyxNQUFNLENBQUNDLElBQUksQ0FBQ0MsUUFBUSxDQUFFUixLQUFLLENBQUNTLE9BQU8sQ0FBQ0gsTUFBTSxDQUFDQyxJQUFJLENBQUNDLFFBQVEsQ0FBQyxDQUNwRixDQUFDLEdBQU0sQ0FBQyxDQUVSLGNBQWUsQ0FBQVQsS0FBSyxDQUFDRixjQUFjLENBQUUsQ0FDakNhLElBQUlBLENBQUUsQ0FDRkMsV0FBVyxDQUFYQSxDQUFXLENBQUcsQ0FDbEIsQ0FBQyxDQUFHLENBQUMsQ0FBQyxDQUFFLENBQ0osTUFBTyxLQUFJLENBQUNDLE9BQU8sQ0FDZixJQUFJLENBQUNDLFFBQVEsQ0FBQyxNQUFNLENBQUUsQ0FDbEJGLFdBQVcsQ0FBWEEsQ0FDSixDQUFDLENBQUMsQ0FDRixJQUNSLENBQUMsQ0FDREcsVUFBVUEsQ0FBRSxDQUNSQyxHQUFHLENBQUhBLENBQUcsQ0FBRyxFQUNWLENBQUMsQ0FBRyxDQUFDLENBQUMsQ0FBRSxDQUNKLEdBQUksQ0FBQUMsQ0FBWSxDQUNaQyxDQUFnQixJQUFXLENBQzNCQyxDQUF5QixDQUFHLElBQUksQ0FBQ0MsK0JBQStCLENBQUNKLENBQUcsQ0FBQyxDQUVwRUcsQ0FBeUIsR0FDMUJBLENBQXlCLENBQUcsR0FBSSxDQUFBRSxPQUFTLENBQ3pDLElBQUksQ0FBQ0QsK0JBQStCLENBQUNKLENBQUcsQ0FBQyxDQUFHRyxDQUF5QixFQUd6RSxJQUFLLEtBQU0sQ0FBQUcsQ0FBTSxHQUFJLEtBQUksQ0FBQ0MsUUFBUSxDQUFFLENBQ2hDLEtBQU0sQ0FBQUMsQ0FBVSxDQUFHTCxDQUF5QixDQUFDTSxHQUFHLENBQUNILENBQU0sQ0FBQyxDQUV4RCxHQUFJRSxDQUFVLENBQ05BLENBQVUsQ0FBR04sQ0FBZ0IsR0FDN0JELENBQVksQ0FBR0ssQ0FBTSxDQUNyQkosQ0FBZ0IsQ0FBR00sQ0FBVSxNQUU5QixDQUNIUCxDQUFZLENBQUdLLENBQU0sQ0FDckIsS0FDSixDQUNKLENBR0EsTUFEQSxDQUFBSCxDQUF5QixDQUFDTyxHQUFHLENBQUNULENBQVksQ0FBRVUsSUFBSSxDQUFDQyxHQUFHLENBQUMsQ0FBQyxDQUFDLENBQ2hEWCxDQUNYLENBQUMsQ0FDRFksSUFBSUEsQ0FBRSxDQUNGQyxPQUFPLENBQVBBLENBQU8sQ0FDUEMsRUFBRSxDQUFGQSxDQUFFLENBQUcsSUFBSSxDQUFDUixRQUNkLENBQUMsQ0FBRSxDQU9DLE1BTkssQ0FBQVMsS0FBSyxDQUFDQyxPQUFPLENBQUNGLENBQUUsQ0FBQyxHQUNsQkEsQ0FBRSxDQUFHLENBQ0RBLENBQUUsQ0FDTCxFQUdFRyxPQUFPLENBQUNDLEdBQUcsQ0FBQ0osQ0FBRSxDQUFDSyxHQUFHLENBQUNMLENBQUUsRUFBSSxHQUFJLENBQUFHLE9BQU8sQ0FBQyxDQUFDRyxDQUFPLENBQUVDLENBQU0sR0FBSyxDQUM3RCxLQUFNLENBQUFoQixDQUFNLENBQWlCLFFBQVEsRUFBdEIsTUFBTyxDQUFBUyxDQUFlLENBQ2pDLElBQUksQ0FBQ1EsV0FBVyxDQUFDUixDQUFFLENBQUMsQ0FDcEJBLENBQUUsQ0FFRlQsQ0FBTSxFQUFJQSxDQUFNLENBQUNPLElBQUksQ0FDckJQLENBQU0sQ0FBQ08sSUFBSSxDQUFDQyxDQUFPLENBQUVVLENBQUssRUFBSSxDQUN0QkEsQ0FBSyxDQUNMRixDQUFNLENBQUN6QyxNQUFNLENBQUMsQ0FDVjRDLE9BQU8sQ0FBRSxDQUNMWCxPQUFPLENBQVBBLENBQU8sQ0FDUFksUUFBUSxDQUFFcEIsQ0FBTSxDQUFDcUIsRUFDckIsQ0FBQyxDQUNESCxLQUFLLENBQUxBLENBQUssQ0FDTFYsT0FBTyxDQUFFLGlDQUNiLENBQUMsQ0FBQyxDQUFDLENBRUhPLENBQU8sQ0FBQyxDQUVoQixDQUFDLENBQUMsQ0FFRkMsQ0FBTSxDQUFDekMsTUFBTSxDQUFDLENBQ1Y0QyxPQUFPLENBQUUsQ0FDTFgsT0FBTyxDQUFQQSxDQUFPLENBQ1BDLEVBQUUsQ0FBRkEsQ0FDSixDQUFDLENBQ0RELE9BQU8sQ0FBRSw2QkFDYixDQUFDLENBQUMsQ0FFVixDQUFDLENBQUMsQ0FBQyxDQUNQLENBQUMsQ0FDRGMsUUFBUUEsQ0FBQSxDQUFJLENBQ1IsTUFBTyxLQUFJLENBQUM5QixRQUFRLENBQUMsVUFBVSxDQUNuQyxDQUFDLENBQ0QsR0FBSSxDQUFBK0IsVUFBVUEsQ0FBQSxDQUFJLENBQ2QsTUFBTyxLQUFJLENBQUNOLFdBQ2hCLENBQUMsQ0FDRCxHQUFJLENBQUFPLE9BQU9BLENBQUEsQ0FBSSxDQUNYLE1BQU8sS0FBSSxDQUFDdkIsUUFDaEIsQ0FBQyxDQUNEd0IsVUFBVUEsQ0FBRSxDQUNSekIsTUFBTSxDQUFOQSxDQUNKLENBQUMsQ0FBRSxDQUNDLE1BQU8sS0FBSSxDQUFDUixRQUFRLENBQUMsV0FBVyxDQUFFLENBQzlCUSxNQUFNLENBQU5BLENBQ0osQ0FBQyxDQUNMLENBQUMsQ0FDRDBCLFFBQVFBLENBQUUsR0FBR0MsQ0FBSSxDQUFFLE9BQ1gsS0FBSSxDQUFDcEMsT0FBTyxFQUNaLElBQUksQ0FBQ3FDLFVBQVUsQ0FBQyxrQkFBa0IsQ0FBRSxVQUFVLENBQUMsS0FDL0MsS0FBSSxDQUFDTixRQUFRLENBQUMsQ0FBQyxPQUluQixJQUFJLENBQUMvQixPQUFPLE9BQVksQ0FFeEJzQyxNQUFNLENBQUNDLElBQUksQ0FBQyxJQUFJLENBQUNDLDJCQUEyQixDQUFDLENBQUNDLE9BQU8sQ0FBQ0MsQ0FBUyxFQUFJLENBQy9EM0QsUUFBUSxDQUFDNEQsY0FBYyxDQUFDRCxDQUFTLENBQUUsSUFBSSxDQUFDRiwyQkFBMkIsQ0FBQ0UsQ0FBUyxDQUFDLENBQ2xGLENBQUMsQ0FBQyxDQUVGLElBQUksQ0FBQ0YsMkJBQTJCLE9BQVksQ0FFNUMsSUFBSSxDQUFDakMsK0JBQStCLE9BQVksQ0FFaERxQyxPQUFPLENBQUNDLEtBQUssQ0FBQzVELGNBQWMsQ0FBQzZELFNBQVMsQ0FBQ1gsUUFBUSxDQUFFLElBQUksQ0FBRUMsQ0FBSSxDQUFDLENBQ2hFLENBQUMsQ0FDRFcsZUFBZUEsQ0FBRSxDQUNiQyxJQUFJLENBQUUsQ0FDRnZDLE1BQU0sQ0FBTkEsQ0FDSixDQUNKLENBQUMsQ0FBRSxDQUNDLElBQUksQ0FBQ2lCLFdBQVcsQ0FBQ2pCLENBQU0sQ0FBQ3FCLEVBQUUsQ0FBQyxDQUFHckIsQ0FBTSxDQUNwQyxJQUFJLENBQUNDLFFBQVEsQ0FBRzRCLE1BQU0sQ0FBQ1csTUFBTSxDQUFDLElBQUksQ0FBQ3ZCLFdBQVcsQ0FDbEQsQ0FBQyxDQUNEd0IsVUFBVUEsQ0FBRSxDQUNSRixJQUFJLENBQUUsQ0FDRmpELFdBQVcsQ0FBWEEsQ0FDSixDQUNKLENBQUMsQ0FBRSxNQUNzQixDQUFDLENBQWZBLENBQWUsRUFDbEJULGFBQWEsQ0FBQzZELElBQUksQ0FBQyx1QkFBdUIsQ0FBQyxDQUUzQ3BFLFFBQVEsQ0FBQ2UsSUFBSSxDQUFDLENBQUMsQ0FDZkMsQ0FBVyxFQUFJLENBRXZCLENBQUMsQ0FDRHFELGtCQUFrQkEsQ0FBRSxDQUNoQkosSUFBSSxDQUFFLENBQ0Z2QyxNQUFNLENBQU5BLENBQ0osQ0FDSixDQUFDLENBQUUsQ0FDQyxNQUFPLEtBQUksQ0FBQ2lCLFdBQVcsQ0FBQ2pCLENBQU0sQ0FBQ3FCLEVBQUUsQ0FBQyxDQUNsQyxJQUFJLENBQUNwQixRQUFRLENBQUc0QixNQUFNLENBQUNXLE1BQU0sQ0FBQyxJQUFJLENBQUN2QixXQUFXLENBQ2xELENBQUMsQ0FDRDJCLGNBQWNBLENBQUEsQ0FBSSxDQVNkLE1BUkEsQ0FBQS9ELGFBQWEsQ0FBQzZELElBQUksQ0FBQyx1QkFBdUIsQ0FBQyxDQUUzQyxJQUFJLENBQUNuRCxPQUFPLEdBQVEsQ0FFcEJqQixRQUFRLENBQUN1RSxVQUFVLENBQUMsSUFBTSxDQUN0QixJQUFJLENBQUNyRCxRQUFRLENBQUMsa0JBQWtCLENBQ3BDLENBQUMsQ0FBQyxDQUVLLElBQ1gsQ0FBQyxDQUNEc0Qsc0JBQXNCQSxDQUFBLENBQUksQ0FDdEJqRSxhQUFhLENBQUM2RCxJQUFJLENBQUMsMkJBQTJCLENBQ2xELENBQUMsQ0FDREssc0JBQXNCQSxDQUFFLENBQ3BCUixJQUFJLENBQUUsQ0FDRnZDLE1BQU0sQ0FBTkEsQ0FDSixDQUNKLENBQUMsQ0FBRSxDQUNDbkIsYUFBYSxDQUFDNkQsSUFBSSxDQUFDLENBQ2Z0QixRQUFRLENBQUVwQixDQUFNLENBQUNxQixFQUNyQixDQUFDLENBQUUscUJBQXFCLENBQUMsQ0FFekIsSUFBSSxDQUFDMkIsYUFBYSxDQUFDLENBQ2ZoRCxNQUFNLENBQU5BLENBQ0osQ0FBQyxDQUNMLENBQUMsQ0FDRGlELGlCQUFpQkEsQ0FBRSxDQUNmVixJQUFJLENBQUUsQ0FDRnJCLEtBQUssQ0FBTEEsQ0FBSyxDQUNMbEIsTUFBTSxDQUFOQSxDQUNKLENBQ0osQ0FBQyxDQUFFLENBQ0NuQixhQUFhLENBQUNxRSxLQUFLLENBQUMsQ0FDaEJoQyxLQUFLLENBQUUzQyxNQUFNLENBQUMsQ0FDVjRDLE9BQU8sQ0FBRSxDQUNMQyxRQUFRLENBQUVwQixDQUFNLENBQUNxQixFQUNyQixDQUFDLENBQ0RILEtBQUssQ0FBTEEsQ0FBSyxDQUNMVixPQUFPLENBQUUsY0FDYixDQUFDLENBQUMsQ0FDRlksUUFBUSxDQUFFcEIsQ0FBTSxDQUFDcUIsRUFDckIsQ0FBQyxDQUFFLGNBQWMsQ0FBQyxDQUVsQixJQUFJLENBQUMyQixhQUFhLENBQUMsQ0FDZmhELE1BQU0sQ0FBTkEsQ0FDSixDQUFDLENBQ0wsQ0FBQyxDQUNEbUQsZ0JBQWdCQSxDQUFFLENBQ2RaLElBQUksQ0FBRSxDQUNGYSxJQUFJLENBQUpBLENBQUksQ0FDSkMsTUFBTSxDQUFOQSxDQUFNLENBQ05yRCxNQUFNLENBQU5BLENBQ0osQ0FDSixDQUFDLENBQUUsQ0FDS0EsQ0FBTSxDQUFDc0QscUJBQXFCLENBQzVCekUsYUFBYSxDQUFDNkQsSUFBSSxDQUFDLENBQ2ZVLElBQUksQ0FBSkEsQ0FBSSxDQUNKQyxNQUFNLENBQU5BLENBQU0sQ0FDTmpDLFFBQVEsQ0FBRXBCLENBQU0sQ0FBQ3FCLEVBQ3JCLENBQUMsQ0FBRSwyQkFBMkIsQ0FBQyxDQUN4QitCLENBQUksQ0FDWHZFLGFBQWEsQ0FBQ3FFLEtBQUssQ0FBQyxDQUNoQkUsSUFBSSxDQUFKQSxDQUFJLENBQ0pDLE1BQU0sQ0FBTkEsQ0FBTSxDQUNOakMsUUFBUSxDQUFFcEIsQ0FBTSxDQUFDcUIsRUFDckIsQ0FBQyxDQUFFLDBCQUEwQixDQUFDLENBRTlCeEMsYUFBYSxDQUFDcUMsS0FBSyxDQUFDLENBQ2hCa0MsSUFBSSxDQUFKQSxDQUFJLENBQ0pDLE1BQU0sQ0FBTkEsQ0FBTSxDQUNOakMsUUFBUSxDQUFFcEIsQ0FBTSxDQUFDcUIsRUFDckIsQ0FBQyxDQUFFLGFBQWEsQ0FBQyxDQUdyQixJQUFJLENBQUMyQixhQUFhLENBQUMsQ0FDZmhELE1BQU0sQ0FBTkEsQ0FDSixDQUFDLENBQUMsQ0FFRSxJQUFJLENBQUNULE9BQU8sR0FDWlYsYUFBYSxDQUFDNkQsSUFBSSxDQUFDLHVCQUF1QixDQUFDLENBRTNDLElBQUksQ0FBQ3JELElBQUksQ0FBQyxDQUFDLENBRW5CLENBQUMsQ0FDRGtFLGdCQUFnQkEsQ0FBRSxDQUNkaEIsSUFBSSxDQUFFLENBQ0Z2QyxNQUFNLENBQU5BLENBQ0osQ0FDSixDQUFDLENBQUUsQ0FDQ25CLGFBQWEsQ0FBQzZELElBQUksQ0FBQyxDQUNmdEIsUUFBUSxDQUFFcEIsQ0FBTSxDQUFDcUIsRUFDckIsQ0FBQyxDQUFFLHFCQUFxQixDQUFDLENBRXpCckIsQ0FBTSxDQUFDd0QsRUFBRSxDQUFDLE9BQU8sQ0FBRXRDLENBQUssRUFBSSxDQUN4QixJQUFJLENBQUMxQixRQUFRLENBQUMsYUFBYSxDQUFFLENBQ3pCMEIsS0FBSyxDQUFFM0MsTUFBTSxDQUFDLENBQ1Y0QyxPQUFPLENBQUUsQ0FDTG5CLE1BQU0sQ0FBTkEsQ0FDSixDQUFDLENBQ0RrQixLQUFLLENBQUxBLENBQUssQ0FDTFYsT0FBTyxDQUFFLGlCQUNiLENBQUMsQ0FBQyxDQUNGUixNQUFNLENBQU5BLENBQ0osQ0FBQyxDQUNMLENBQUMsQ0FDTCxDQUFDLENBQ0R5RCxxQkFBcUJBLENBQUUsQ0FDbkJsQixJQUFJLENBQUUsQ0FDRm1CLE9BQU8sQ0FBUEEsQ0FBTyxDQUNQMUQsTUFBTSxDQUFOQSxDQUNKLENBQ0osQ0FBQyxDQUFFLENBQ0NuQixhQUFhLENBQUM2RCxJQUFJLENBQUMsQ0FDZmdCLE9BQU8sQ0FBRUEsQ0FBTyxDQUFDQSxPQUFPLENBQ3hCQyxJQUFJLENBQUVELENBQU8sQ0FBQ0MsSUFBSSxDQUNsQnZDLFFBQVEsQ0FBRXBCLENBQU0sQ0FBQ3FCLEVBQ3JCLENBQUMsQ0FBRSxxQkFBcUIsQ0FDNUIsQ0FBQyxDQUNEdUMsbUJBQW1CQSxDQUFFQyxDQUFLLENBQUUsQ0FDeEIsR0FBSUEsQ0FBSyxDQUFDdEIsSUFBSSxDQUFDL0IsT0FBTyxFQUFJcUQsQ0FBSyxDQUFDdEIsSUFBSSxDQUFDL0IsT0FBTyxDQUFDc0QsSUFBSSxDQUFFLENBQy9DLEtBQU0sQ0FBQUMsQ0FBTSxDQUFHLElBQUksQ0FBQyx1QkFBdUJGLENBQUssQ0FBQ3RCLElBQUksQ0FBQy9CLE9BQU8sQ0FBQ3NELElBQUksRUFBRSxDQUFDLENBRS9DLFVBQVUsRUFBNUIsTUFBTyxDQUFBQyxDQUFxQixFQUM1QjVCLE9BQU8sQ0FBQ0MsS0FBSyxDQUFDMkIsQ0FBTSxDQUFFLElBQUksQ0FBRSxDQUN4QkYsQ0FBSyxDQUNSLENBRVQsQ0FDSixDQUFDLENBQ0RHLGtCQUFrQkEsQ0FBRSxDQUNoQnpCLElBQUksQ0FBRSxDQUNGdkMsTUFBTSxDQUFOQSxDQUNKLENBQ0osQ0FBQyxDQUFFLENBQ0NuQixhQUFhLENBQUM2RCxJQUFJLENBQUMsQ0FDZnRCLFFBQVEsQ0FBRXBCLENBQU0sQ0FBQ3FCLEVBQ3JCLENBQUMsQ0FBRSxzQkFBc0IsQ0FBQyxDQUUxQixLQUFNLENBQUE0QyxDQUFRLENBQUd6RCxDQUFPLEVBQUksQ0FDUixPQUFPLEdBQW5CQSxDQUFtQixHQUNuQlIsQ0FBTSxDQUFDa0MsY0FBYyxDQUFDLFNBQVMsQ0FBRStCLENBQVEsQ0FBQyxDQUUxQyxJQUFJLENBQUN6RSxRQUFRLENBQUMsYUFBYSxDQUFFLENBQ3pCUSxNQUFNLENBQU5BLENBQ0osQ0FBQyxDQUFDLENBRVYsQ0FBQyxDQUVEQSxDQUFNLENBQUN3RCxFQUFFLENBQUMsU0FBUyxDQUFFUyxDQUFRLENBQ2pDLENBQUMsQ0FDREMsaUJBQWlCQSxDQUFFLENBQ2YzQixJQUFJLENBQUUsQ0FDRnZDLE1BQU0sQ0FBTkEsQ0FDSixDQUNKLENBQUMsQ0FBRSxDQUNDbkIsYUFBYSxDQUFDNkQsSUFBSSxDQUFDLENBQ2Z0QixRQUFRLENBQUVwQixDQUFNLENBQUNxQixFQUNyQixDQUFDLENBQUUseUJBQXlCLENBQUMsQ0FFN0IsSUFBSSxDQUFDSSxVQUFVLENBQUMsQ0FDWnpCLE1BQU0sQ0FBTkEsQ0FDSixDQUFDLENBQ0wsQ0FBQyxDQUNEbUUsS0FBS0EsQ0FBRSxHQUFHeEMsQ0FBSSxDQUFFLENBK0NaLE1BOUNBLEtBQUksQ0FBQ3BDLE9BQU8sR0FBTyxDQUVuQixJQUFJLENBQUN3QywyQkFBMkIsQ0FBRyxDQUMvQmMsVUFBVSxDQUFFN0MsQ0FBTSxFQUFJLENBQ2xCLElBQUksQ0FBQ1IsUUFBUSxDQUFDLGtCQUFrQixDQUFFLENBQzlCUSxNQUFNLENBQU5BLENBQ0osQ0FBQyxDQUNMLENBQUMsQ0FDRG9FLElBQUksQ0FBRUEsQ0FBQ3BFLENBQU0sQ0FBRW9ELENBQUksQ0FBRUMsQ0FBTSxHQUFLLENBQzVCLElBQUksQ0FBQzdELFFBQVEsQ0FBQyxZQUFZLENBQUUsQ0FDeEI0RCxJQUFJLENBQUpBLENBQUksQ0FDSkMsTUFBTSxDQUFOQSxDQUFNLENBQ05yRCxNQUFNLENBQU5BLENBQ0osQ0FBQyxDQUNMLENBQUMsQ0FDRFgsSUFBSSxDQUFFVyxDQUFNLEVBQUksQ0FDWixJQUFJLENBQUNSLFFBQVEsQ0FBQyxZQUFZLENBQUUsQ0FDeEJRLE1BQU0sQ0FBTkEsQ0FDSixDQUFDLENBQ0wsQ0FBQyxDQUNEcUUsU0FBUyxDQUFFQSxDQUFDckUsQ0FBTSxDQUFFMEQsQ0FBTyxHQUFLLENBQzVCLElBQUksQ0FBQ2xFLFFBQVEsQ0FBQyxpQkFBaUIsQ0FBRSxDQUM3QmtFLE9BQU8sQ0FBUEEsQ0FBTyxDQUNQMUQsTUFBTSxDQUFOQSxDQUNKLENBQUMsQ0FDTCxDQUFDLENBQ0RRLE9BQU8sQ0FBRUEsQ0FBQ1IsQ0FBTSxDQUFFUSxDQUFPLENBQUU4RCxDQUFNLEdBQUssQ0FDbEMsSUFBSSxDQUFDOUUsUUFBUSxDQUFDLGVBQWUsQ0FBRSxDQUMzQjhFLE1BQU0sQ0FBTkEsQ0FBTSxDQUNOOUQsT0FBTyxDQUFQQSxDQUFPLENBQ1BSLE1BQU0sQ0FBTkEsQ0FDSixDQUFDLENBQ0wsQ0FBQyxDQUNEdUUsTUFBTSxDQUFFdkUsQ0FBTSxFQUFJLENBQ2QsSUFBSSxDQUFDUixRQUFRLENBQUMsY0FBYyxDQUFFLENBQzFCUSxNQUFNLENBQU5BLENBQ0osQ0FBQyxDQUNMLENBQ0osQ0FBQyxDQUVELElBQUksQ0FBQ2lCLFdBQVcsQ0FBR1ksTUFBTSxDQUFDMkMsTUFBTSxDQUFDLElBQUksQ0FBQyxDQUV0QyxJQUFJLENBQUN2RSxRQUFRLENBQUcsRUFBRSxDQUVsQixJQUFJLENBQUNILCtCQUErQixDQUFHK0IsTUFBTSxDQUFDMkMsTUFBTSxDQUFDLElBQUksQ0FBQyxDQUVuRHJDLE9BQU8sQ0FBQ0MsS0FBSyxDQUFDNUQsY0FBYyxDQUFDNkQsU0FBUyxDQUFDOEIsS0FBSyxDQUFFLElBQUksQ0FBRXhDLENBQUksQ0FDbkUsQ0FBQyxDQUNEOEMsV0FBV0EsQ0FBRSxHQUFHOUMsQ0FBSSxDQUFFLENBQ2xCOUMsYUFBYSxDQUFDNkQsSUFBSSxDQUFDLHNCQUFzQixDQUFDLENBRTFDLEtBQU0sQ0FBQyxDQUNIZ0MsVUFBVSxDQUFWQSxDQUFVLENBQ1ZDLFlBQVksQ0FBWkEsQ0FBWSxDQUNaQyxZQUFZLENBQVpBLENBQVksQ0FDWkMsV0FBVyxDQUFYQSxDQUNKLENBQUMsQ0FBRyxDQUFDLENBQUMsQ0FBQyxDQUFHbEQsQ0FBSSxDQUVkckQsUUFBUSxDQUFDd0csWUFBWSxDQUFDLENBQ2xCbkQsSUFBSSxDQUF3QixXQUFXLEVBQWpDLE1BQU8sQ0FBQStDLENBQTBCLENBQ25DOUYsUUFBUSxDQUFDbUcsSUFBSSxDQUFDQyxLQUFLLENBQUMsQ0FBQyxDQUFDLENBQ3RCTixDQUFVLENBQ2RPLElBQUksQ0FBMEIsV0FBVyxFQUFuQyxNQUFPLENBQUFOLENBQTRCLENBQ3JDL0YsUUFBUSxDQUFDbUcsSUFBSSxDQUFDLENBQUMsQ0FBQyxDQUNoQkosQ0FBWSxDQUNoQk8sTUFBTSxDQUEwQixXQUFXLEVBQW5DLE1BQU8sQ0FBQU4sQ0FBNEIsRUFFdkNBLENBQVksQ0FDaEJPLEtBQUssQ0FBRU4sQ0FDWCxDQUFDLENBQUMsQ0FFRmhELE1BQU0sQ0FBQ0MsSUFBSSxDQUFDLElBQUksQ0FBQ0MsMkJBQTJCLENBQUMsQ0FBQ0MsT0FBTyxDQUFDQyxDQUFTLEVBQUksQ0FDL0QzRCxRQUFRLENBQUNrRixFQUFFLENBQUN2QixDQUFTLENBQUUsSUFBSSxDQUFDRiwyQkFBMkIsQ0FBQ0UsQ0FBUyxDQUFDLENBQ3RFLENBQUMsQ0FDTCxDQUFDLENBQ0RlLGFBQWFBLENBQUUsQ0FDWGhELE1BQU0sQ0FBTkEsQ0FDSixDQUFDLENBQUUsQ0FPQyxNQU5JLENBQUFBLENBQU0sQ0FBQ3FCLEVBQUUsR0FBSSxLQUFJLENBQUNKLFdBQVcsRUFDN0IsSUFBSSxDQUFDekIsUUFBUSxDQUFDLGNBQWMsQ0FBRSxDQUMxQlEsTUFBTSxDQUFOQSxDQUNKLENBQUMsQ0FBQyxDQUdDLElBQ1gsQ0FDSixDQUFDLENBQUUsQ0FDQ29GLE9BQU8sQ0FBRSxDQUNMQyxTQUFTLENBQUUsQ0FDUEMsZ0JBQWdCLENBQUUsaUJBQ3RCLENBQUMsQ0FDRGpHLElBQUksQ0FBRSxDQUNGaUcsZ0JBQWdCLENBQUUsWUFDdEIsQ0FBQyxDQUNEQyxZQUFZLENBQUUsQ0FDVkQsZ0JBQWdCLENBQUUsb0JBQ3RCLENBQUMsQ0FDRGhFLFFBQVEsQ0FBRSxDQUNOZ0UsZ0JBQWdCLENBQUUsZ0JBQWdCLENBQ2xDRSxZQUFZLEdBQ2hCLENBQUMsQ0FDREMsZ0JBQWdCLENBQUUsQ0FDZEgsZ0JBQWdCLENBQUUsd0JBQXdCLENBQzFDSSxXQUFXLEdBQ2YsQ0FBQyxDQUNEQyxnQkFBZ0IsQ0FBRSxDQUNkTCxnQkFBZ0IsQ0FBRSx3QkFDdEIsQ0FBQyxDQUNETSxXQUFXLENBQUUsQ0FDVE4sZ0JBQWdCLENBQUUsbUJBQ3RCLENBQUMsQ0FDRE8sVUFBVSxDQUFFLENBQ1JQLGdCQUFnQixDQUFFLGtCQUN0QixDQUFDLENBQ0RRLFVBQVUsQ0FBRSxDQUNSUixnQkFBZ0IsQ0FBRSxrQkFDdEIsQ0FBQyxDQUNEUyxlQUFlLENBQUUsQ0FDYlQsZ0JBQWdCLENBQUUsdUJBQ3RCLENBQUMsQ0FDRFUsYUFBYSxDQUFFLENBQ1hWLGdCQUFnQixDQUFFLHFCQUN0QixDQUFDLENBQ0RXLFlBQVksQ0FBRSxDQUNWWCxnQkFBZ0IsQ0FBRSxvQkFDdEIsQ0FBQyxDQUNEWSxXQUFXLENBQUUsQ0FDVFosZ0JBQWdCLENBQUUsbUJBQ3RCLENBQ0osQ0FDSixDQUFDLENBQUMiLCJpZ25vcmVMaXN0IjpbXX0=
|
|
1
|
+
import _Backoff from 'isotropic-backoff';
|
|
2
|
+
import _cluster from 'node:cluster';
|
|
3
|
+
import _Error from 'isotropic-error';
|
|
4
|
+
import _Initializable from 'isotropic-initializable';
|
|
5
|
+
import _logger from 'isotropic-logger';
|
|
6
|
+
import _make from 'isotropic-make';
|
|
7
|
+
import _path from 'node:path';
|
|
8
|
+
import _process from 'node:process';
|
|
9
|
+
|
|
10
|
+
const _moduleLogger = _logger.child({
|
|
11
|
+
module: _path.basename(import.meta.filename, _path.extname(import.meta.filename))
|
|
12
|
+
});
|
|
13
|
+
|
|
14
|
+
export default _make('ClusterPrimary', _Initializable, {
|
|
15
|
+
fork ({
|
|
16
|
+
workerCount = 1
|
|
17
|
+
} = {}) {
|
|
18
|
+
return this._active ?
|
|
19
|
+
this._publish('fork', {
|
|
20
|
+
workerCount
|
|
21
|
+
}) :
|
|
22
|
+
this;
|
|
23
|
+
},
|
|
24
|
+
get restartBackoff () {
|
|
25
|
+
return this._restartBackoff;
|
|
26
|
+
},
|
|
27
|
+
roundRobin ({
|
|
28
|
+
tag = ''
|
|
29
|
+
} = {}) {
|
|
30
|
+
if (!this._workers.length) {
|
|
31
|
+
return;
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
let oldestWorker,
|
|
35
|
+
oldestWorkerTime = Infinity,
|
|
36
|
+
workerTimeByWorkerWeakMap = this._workerTimeByWorkerWeakMapByTag[tag];
|
|
37
|
+
|
|
38
|
+
if (!workerTimeByWorkerWeakMap) {
|
|
39
|
+
workerTimeByWorkerWeakMap = new WeakMap();
|
|
40
|
+
this._workerTimeByWorkerWeakMapByTag[tag] = workerTimeByWorkerWeakMap;
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
for (const worker of this._workers) {
|
|
44
|
+
const workerTime = workerTimeByWorkerWeakMap.get(worker);
|
|
45
|
+
|
|
46
|
+
if (workerTime) {
|
|
47
|
+
if (workerTime < oldestWorkerTime) {
|
|
48
|
+
oldestWorker = worker;
|
|
49
|
+
oldestWorkerTime = workerTime;
|
|
50
|
+
}
|
|
51
|
+
} else {
|
|
52
|
+
oldestWorker = worker;
|
|
53
|
+
break;
|
|
54
|
+
}
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
this._roundRobinCounter += 1;
|
|
58
|
+
|
|
59
|
+
workerTimeByWorkerWeakMap.set(oldestWorker, this._roundRobinCounter);
|
|
60
|
+
|
|
61
|
+
return oldestWorker;
|
|
62
|
+
},
|
|
63
|
+
send ({
|
|
64
|
+
message,
|
|
65
|
+
to = this._workers
|
|
66
|
+
}) {
|
|
67
|
+
if (!Array.isArray(to)) {
|
|
68
|
+
to = [
|
|
69
|
+
to
|
|
70
|
+
];
|
|
71
|
+
}
|
|
72
|
+
|
|
73
|
+
return Promise.all(to.map(to => new Promise((resolve, reject) => {
|
|
74
|
+
const worker = typeof to === 'number' ?
|
|
75
|
+
this._workerById[to] :
|
|
76
|
+
to;
|
|
77
|
+
|
|
78
|
+
if (worker && worker.send) {
|
|
79
|
+
worker.send(message, error => {
|
|
80
|
+
if (error) {
|
|
81
|
+
reject(_Error({
|
|
82
|
+
details: {
|
|
83
|
+
message,
|
|
84
|
+
workerId: worker.id
|
|
85
|
+
},
|
|
86
|
+
error,
|
|
87
|
+
message: 'Error sending message to worker'
|
|
88
|
+
}));
|
|
89
|
+
} else {
|
|
90
|
+
resolve();
|
|
91
|
+
}
|
|
92
|
+
});
|
|
93
|
+
} else {
|
|
94
|
+
reject(_Error({
|
|
95
|
+
details: {
|
|
96
|
+
message,
|
|
97
|
+
to
|
|
98
|
+
},
|
|
99
|
+
message: 'Invalid message destination'
|
|
100
|
+
}));
|
|
101
|
+
}
|
|
102
|
+
})));
|
|
103
|
+
},
|
|
104
|
+
shutDown () {
|
|
105
|
+
return this._publish('shutDown');
|
|
106
|
+
},
|
|
107
|
+
get workerById () {
|
|
108
|
+
return this._workerById;
|
|
109
|
+
},
|
|
110
|
+
get workers () {
|
|
111
|
+
return this._workers;
|
|
112
|
+
},
|
|
113
|
+
_addWorker ({
|
|
114
|
+
worker
|
|
115
|
+
}) {
|
|
116
|
+
return this._publish('addWorker', {
|
|
117
|
+
worker
|
|
118
|
+
});
|
|
119
|
+
},
|
|
120
|
+
_destroy (...args) {
|
|
121
|
+
if (this._active) {
|
|
122
|
+
this._onceAfter('shutDownComplete', '_destroy');
|
|
123
|
+
this.shutDown();
|
|
124
|
+
return;
|
|
125
|
+
}
|
|
126
|
+
|
|
127
|
+
this._active = void null;
|
|
128
|
+
|
|
129
|
+
Object.keys(this._clusterListenerByEventName).forEach(eventName => {
|
|
130
|
+
_cluster.removeListener(eventName, this._clusterListenerByEventName[eventName]);
|
|
131
|
+
});
|
|
132
|
+
|
|
133
|
+
this._clusterListenerByEventName = void null;
|
|
134
|
+
|
|
135
|
+
if (this._ownRestartBackoff) {
|
|
136
|
+
this._restartBackoff.destroy();
|
|
137
|
+
}
|
|
138
|
+
|
|
139
|
+
this._ownRestartBackoff = void null;
|
|
140
|
+
|
|
141
|
+
this._restartAbortController = void null;
|
|
142
|
+
|
|
143
|
+
this._restartBackoff = void null;
|
|
144
|
+
|
|
145
|
+
this._workerTimeByWorkerWeakMapByTag = void null;
|
|
146
|
+
|
|
147
|
+
Reflect.apply(_Initializable.prototype._destroy, this, args);
|
|
148
|
+
},
|
|
149
|
+
_eventAddWorker ({
|
|
150
|
+
data: {
|
|
151
|
+
worker
|
|
152
|
+
}
|
|
153
|
+
}) {
|
|
154
|
+
this._workerById[worker.id] = worker;
|
|
155
|
+
this._workers = Object.values(this._workerById);
|
|
156
|
+
},
|
|
157
|
+
_eventFork ({
|
|
158
|
+
data: {
|
|
159
|
+
workerCount
|
|
160
|
+
}
|
|
161
|
+
}) {
|
|
162
|
+
while (workerCount > 0) {
|
|
163
|
+
_moduleLogger.info('Starting a new worker');
|
|
164
|
+
|
|
165
|
+
_cluster.fork();
|
|
166
|
+
workerCount -= 1;
|
|
167
|
+
}
|
|
168
|
+
},
|
|
169
|
+
_eventRemoveWorker ({
|
|
170
|
+
data: {
|
|
171
|
+
worker
|
|
172
|
+
}
|
|
173
|
+
}) {
|
|
174
|
+
delete this._workerById[worker.id];
|
|
175
|
+
this._workers = Object.values(this._workerById);
|
|
176
|
+
},
|
|
177
|
+
_eventRestartGiveUp () {
|
|
178
|
+
_moduleLogger.fatal('Worker restart attempts exhausted with no remaining workers; giving up');
|
|
179
|
+
|
|
180
|
+
this.shutDown();
|
|
181
|
+
},
|
|
182
|
+
_eventShutDown () {
|
|
183
|
+
_moduleLogger.info('Shutting down workers');
|
|
184
|
+
|
|
185
|
+
this._active = false;
|
|
186
|
+
|
|
187
|
+
this._restartAbortController?.abort();
|
|
188
|
+
|
|
189
|
+
_cluster.disconnect(() => {
|
|
190
|
+
this._publish('shutDownComplete');
|
|
191
|
+
});
|
|
192
|
+
|
|
193
|
+
return this;
|
|
194
|
+
},
|
|
195
|
+
_eventShutDownComplete () {
|
|
196
|
+
_moduleLogger.info('Worker shut down complete');
|
|
197
|
+
},
|
|
198
|
+
_eventWorkerDisconnect ({
|
|
199
|
+
data: {
|
|
200
|
+
worker
|
|
201
|
+
}
|
|
202
|
+
}) {
|
|
203
|
+
_moduleLogger.info({
|
|
204
|
+
workerId: worker.id
|
|
205
|
+
}, 'Worker disconnected');
|
|
206
|
+
|
|
207
|
+
this._removeWorker({
|
|
208
|
+
worker
|
|
209
|
+
});
|
|
210
|
+
},
|
|
211
|
+
_eventWorkerError ({
|
|
212
|
+
data: {
|
|
213
|
+
error,
|
|
214
|
+
worker
|
|
215
|
+
}
|
|
216
|
+
}) {
|
|
217
|
+
_moduleLogger.fatal({
|
|
218
|
+
error: _Error({
|
|
219
|
+
details: {
|
|
220
|
+
workerId: worker.id
|
|
221
|
+
},
|
|
222
|
+
error,
|
|
223
|
+
message: 'Worker error'
|
|
224
|
+
}),
|
|
225
|
+
workerId: worker.id
|
|
226
|
+
}, 'Worker error');
|
|
227
|
+
|
|
228
|
+
this._removeWorker({
|
|
229
|
+
worker
|
|
230
|
+
});
|
|
231
|
+
},
|
|
232
|
+
_eventWorkerExit ({
|
|
233
|
+
data: {
|
|
234
|
+
code,
|
|
235
|
+
signal,
|
|
236
|
+
worker
|
|
237
|
+
}
|
|
238
|
+
}) {
|
|
239
|
+
if (worker.exitedAfterDisconnect) {
|
|
240
|
+
_moduleLogger.info({
|
|
241
|
+
code,
|
|
242
|
+
signal,
|
|
243
|
+
workerId: worker.id
|
|
244
|
+
}, 'Worker exited voluntarily');
|
|
245
|
+
} else if (code) {
|
|
246
|
+
_moduleLogger.fatal({
|
|
247
|
+
code,
|
|
248
|
+
signal,
|
|
249
|
+
workerId: worker.id
|
|
250
|
+
}, 'Worker died unexpectedly');
|
|
251
|
+
} else {
|
|
252
|
+
_moduleLogger.error({
|
|
253
|
+
code,
|
|
254
|
+
signal,
|
|
255
|
+
workerId: worker.id
|
|
256
|
+
}, 'Worker died');
|
|
257
|
+
}
|
|
258
|
+
|
|
259
|
+
this._removeWorker({
|
|
260
|
+
worker
|
|
261
|
+
});
|
|
262
|
+
|
|
263
|
+
if (this._active) {
|
|
264
|
+
if (this._restartBackoff && !worker.exitedAfterDisconnect) {
|
|
265
|
+
this._restartWorker();
|
|
266
|
+
} else {
|
|
267
|
+
_moduleLogger.info('Replacing dead worker');
|
|
268
|
+
|
|
269
|
+
this.fork();
|
|
270
|
+
}
|
|
271
|
+
}
|
|
272
|
+
},
|
|
273
|
+
_eventWorkerFork ({
|
|
274
|
+
data: {
|
|
275
|
+
worker
|
|
276
|
+
}
|
|
277
|
+
}) {
|
|
278
|
+
_moduleLogger.info({
|
|
279
|
+
workerId: worker.id
|
|
280
|
+
}, 'Starting new worker');
|
|
281
|
+
|
|
282
|
+
worker.on('error', error => {
|
|
283
|
+
this._publish('workerError', {
|
|
284
|
+
error: _Error({
|
|
285
|
+
details: {
|
|
286
|
+
worker
|
|
287
|
+
},
|
|
288
|
+
error,
|
|
289
|
+
message: 'Error in worker'
|
|
290
|
+
}),
|
|
291
|
+
worker
|
|
292
|
+
});
|
|
293
|
+
});
|
|
294
|
+
},
|
|
295
|
+
_eventWorkerListening ({
|
|
296
|
+
data: {
|
|
297
|
+
address,
|
|
298
|
+
worker
|
|
299
|
+
}
|
|
300
|
+
}) {
|
|
301
|
+
_moduleLogger.info({
|
|
302
|
+
address: address.address,
|
|
303
|
+
port: address.port,
|
|
304
|
+
workerId: worker.id
|
|
305
|
+
}, 'Worker is listening');
|
|
306
|
+
},
|
|
307
|
+
_eventWorkerMessage (event) {
|
|
308
|
+
if (event.data.message && event.data.message.type) {
|
|
309
|
+
const method = this[`_eventWorkerMessage_${event.data.message.type}`];
|
|
310
|
+
|
|
311
|
+
if (typeof method === 'function') {
|
|
312
|
+
Reflect.apply(method, this, [
|
|
313
|
+
event
|
|
314
|
+
]);
|
|
315
|
+
}
|
|
316
|
+
}
|
|
317
|
+
},
|
|
318
|
+
_eventWorkerOnline ({
|
|
319
|
+
data: {
|
|
320
|
+
worker
|
|
321
|
+
}
|
|
322
|
+
}) {
|
|
323
|
+
_moduleLogger.info({
|
|
324
|
+
workerId: worker.id
|
|
325
|
+
}, 'Worker is now online');
|
|
326
|
+
|
|
327
|
+
const listener = message => {
|
|
328
|
+
if (message === 'ready') {
|
|
329
|
+
worker.removeListener('message', listener);
|
|
330
|
+
|
|
331
|
+
this._publish('workerReady', {
|
|
332
|
+
worker
|
|
333
|
+
});
|
|
334
|
+
}
|
|
335
|
+
};
|
|
336
|
+
|
|
337
|
+
worker.on('message', listener);
|
|
338
|
+
},
|
|
339
|
+
_eventWorkerReady ({
|
|
340
|
+
data: {
|
|
341
|
+
worker
|
|
342
|
+
}
|
|
343
|
+
}) {
|
|
344
|
+
_moduleLogger.info({
|
|
345
|
+
workerId: worker.id
|
|
346
|
+
}, 'Worker is ready to work');
|
|
347
|
+
|
|
348
|
+
this._restartBackoff?.succeeded();
|
|
349
|
+
|
|
350
|
+
this._addWorker({
|
|
351
|
+
worker
|
|
352
|
+
});
|
|
353
|
+
},
|
|
354
|
+
_init (...args) {
|
|
355
|
+
this._active = true;
|
|
356
|
+
|
|
357
|
+
this._clusterListenerByEventName = {
|
|
358
|
+
disconnect: worker => {
|
|
359
|
+
this._publish('workerDisconnect', {
|
|
360
|
+
worker
|
|
361
|
+
});
|
|
362
|
+
},
|
|
363
|
+
exit: (worker, code, signal) => {
|
|
364
|
+
this._publish('workerExit', {
|
|
365
|
+
code,
|
|
366
|
+
signal,
|
|
367
|
+
worker
|
|
368
|
+
});
|
|
369
|
+
},
|
|
370
|
+
fork: worker => {
|
|
371
|
+
this._publish('workerFork', {
|
|
372
|
+
worker
|
|
373
|
+
});
|
|
374
|
+
},
|
|
375
|
+
listening: (worker, address) => {
|
|
376
|
+
this._publish('workerListening', {
|
|
377
|
+
address,
|
|
378
|
+
worker
|
|
379
|
+
});
|
|
380
|
+
},
|
|
381
|
+
message: (worker, message, handle) => {
|
|
382
|
+
this._publish('workerMessage', {
|
|
383
|
+
handle,
|
|
384
|
+
message,
|
|
385
|
+
worker
|
|
386
|
+
});
|
|
387
|
+
},
|
|
388
|
+
online: worker => {
|
|
389
|
+
this._publish('workerOnline', {
|
|
390
|
+
worker
|
|
391
|
+
});
|
|
392
|
+
}
|
|
393
|
+
};
|
|
394
|
+
|
|
395
|
+
this._workerById = Object.create(null);
|
|
396
|
+
|
|
397
|
+
this._workers = [];
|
|
398
|
+
|
|
399
|
+
this._roundRobinCounter = 0;
|
|
400
|
+
|
|
401
|
+
this._workerTimeByWorkerWeakMapByTag = Object.create(null);
|
|
402
|
+
|
|
403
|
+
return Reflect.apply(_Initializable.prototype._init, this, args);
|
|
404
|
+
},
|
|
405
|
+
_initialize (...args) {
|
|
406
|
+
_moduleLogger.info('Initializing primary');
|
|
407
|
+
|
|
408
|
+
const [{
|
|
409
|
+
restartBackoff,
|
|
410
|
+
workerArgs,
|
|
411
|
+
workerScript,
|
|
412
|
+
workerSilent,
|
|
413
|
+
workerStdio
|
|
414
|
+
} = {}] = args;
|
|
415
|
+
|
|
416
|
+
if (restartBackoff || typeof restartBackoff === 'undefined') {
|
|
417
|
+
if (typeof restartBackoff?.failed === 'function') {
|
|
418
|
+
this._restartBackoff = restartBackoff;
|
|
419
|
+
} else {
|
|
420
|
+
this._ownRestartBackoff = true;
|
|
421
|
+
this._restartBackoff = _Backoff(restartBackoff);
|
|
422
|
+
}
|
|
423
|
+
|
|
424
|
+
this._restartAbortController = new AbortController();
|
|
425
|
+
}
|
|
426
|
+
|
|
427
|
+
_cluster.setupPrimary({
|
|
428
|
+
args: typeof workerArgs === 'undefined' ?
|
|
429
|
+
_process.argv.slice(2) :
|
|
430
|
+
workerArgs,
|
|
431
|
+
exec: typeof workerScript === 'undefined' ?
|
|
432
|
+
_process.argv[1] :
|
|
433
|
+
workerScript,
|
|
434
|
+
silent: typeof workerSilent === 'undefined' ?
|
|
435
|
+
false :
|
|
436
|
+
workerSilent,
|
|
437
|
+
stdio: workerStdio
|
|
438
|
+
});
|
|
439
|
+
|
|
440
|
+
Object.keys(this._clusterListenerByEventName).forEach(eventName => {
|
|
441
|
+
_cluster.on(eventName, this._clusterListenerByEventName[eventName]);
|
|
442
|
+
});
|
|
443
|
+
},
|
|
444
|
+
_removeWorker ({
|
|
445
|
+
worker
|
|
446
|
+
}) {
|
|
447
|
+
if (worker.id in this._workerById) {
|
|
448
|
+
this._publish('removeWorker', {
|
|
449
|
+
worker
|
|
450
|
+
});
|
|
451
|
+
}
|
|
452
|
+
|
|
453
|
+
return this;
|
|
454
|
+
},
|
|
455
|
+
_restartWorker () {
|
|
456
|
+
this._restartBackoff.failed();
|
|
457
|
+
|
|
458
|
+
if (this._restartBackoff.exhausted) {
|
|
459
|
+
if (this._workers.length) {
|
|
460
|
+
_moduleLogger.error('Worker restart attempts exhausted; continuing with remaining workers');
|
|
461
|
+
} else {
|
|
462
|
+
this._publish('restartGiveUp');
|
|
463
|
+
}
|
|
464
|
+
|
|
465
|
+
return this;
|
|
466
|
+
}
|
|
467
|
+
|
|
468
|
+
const {
|
|
469
|
+
delay
|
|
470
|
+
} = this._restartBackoff;
|
|
471
|
+
|
|
472
|
+
if (delay) {
|
|
473
|
+
_moduleLogger.warn({
|
|
474
|
+
delay
|
|
475
|
+
}, 'Delaying worker replacement');
|
|
476
|
+
|
|
477
|
+
this._restartBackoff.attempt({
|
|
478
|
+
signal: this._restartAbortController.signal
|
|
479
|
+
}).then(() => {
|
|
480
|
+
_moduleLogger.info('Replacing dead worker');
|
|
481
|
+
|
|
482
|
+
this.fork();
|
|
483
|
+
}, () => {
|
|
484
|
+
// The delayed replacement was canceled during shut down.
|
|
485
|
+
});
|
|
486
|
+
} else {
|
|
487
|
+
_moduleLogger.info('Replacing dead worker');
|
|
488
|
+
|
|
489
|
+
this.fork();
|
|
490
|
+
}
|
|
491
|
+
|
|
492
|
+
return this;
|
|
493
|
+
}
|
|
494
|
+
}, {
|
|
495
|
+
_pubsub: {
|
|
496
|
+
addWorker: {
|
|
497
|
+
completeFunction: '_eventAddWorker'
|
|
498
|
+
},
|
|
499
|
+
fork: {
|
|
500
|
+
completeFunction: '_eventFork'
|
|
501
|
+
},
|
|
502
|
+
removeWorker: {
|
|
503
|
+
completeFunction: '_eventRemoveWorker'
|
|
504
|
+
},
|
|
505
|
+
restartGiveUp: {
|
|
506
|
+
completeFunction: '_eventRestartGiveUp'
|
|
507
|
+
},
|
|
508
|
+
shutDown: {
|
|
509
|
+
completeFunction: '_eventShutDown',
|
|
510
|
+
completeOnce: true
|
|
511
|
+
},
|
|
512
|
+
shutDownComplete: {
|
|
513
|
+
completeFunction: '_eventShutDownComplete',
|
|
514
|
+
publishOnce: true
|
|
515
|
+
},
|
|
516
|
+
workerDisconnect: {
|
|
517
|
+
completeFunction: '_eventWorkerDisconnect'
|
|
518
|
+
},
|
|
519
|
+
workerError: {
|
|
520
|
+
completeFunction: '_eventWorkerError'
|
|
521
|
+
},
|
|
522
|
+
workerExit: {
|
|
523
|
+
completeFunction: '_eventWorkerExit'
|
|
524
|
+
},
|
|
525
|
+
workerFork: {
|
|
526
|
+
completeFunction: '_eventWorkerFork'
|
|
527
|
+
},
|
|
528
|
+
workerListening: {
|
|
529
|
+
completeFunction: '_eventWorkerListening'
|
|
530
|
+
},
|
|
531
|
+
workerMessage: {
|
|
532
|
+
completeFunction: '_eventWorkerMessage'
|
|
533
|
+
},
|
|
534
|
+
workerOnline: {
|
|
535
|
+
completeFunction: '_eventWorkerOnline'
|
|
536
|
+
},
|
|
537
|
+
workerReady: {
|
|
538
|
+
completeFunction: '_eventWorkerReady'
|
|
539
|
+
}
|
|
540
|
+
}
|
|
541
|
+
});
|
package/lib/cluster-worker.js
CHANGED
|
@@ -1,2 +1,117 @@
|
|
|
1
|
-
import _cluster from
|
|
2
|
-
|
|
1
|
+
import _cluster from 'node:cluster';
|
|
2
|
+
import _Error from 'isotropic-error';
|
|
3
|
+
import _Initializable from 'isotropic-initializable';
|
|
4
|
+
import _later from 'isotropic-later';
|
|
5
|
+
import _logger from 'isotropic-logger';
|
|
6
|
+
import _make from 'isotropic-make';
|
|
7
|
+
import _path from 'node:path';
|
|
8
|
+
import _process from 'node:process';
|
|
9
|
+
|
|
10
|
+
const _workerId = _cluster.worker && _cluster.worker.id,
|
|
11
|
+
|
|
12
|
+
_moduleLogger = _logger.child({
|
|
13
|
+
module: _path.basename(import.meta.filename, _path.extname(import.meta.filename)),
|
|
14
|
+
workerId: _workerId
|
|
15
|
+
});
|
|
16
|
+
|
|
17
|
+
export default _make('ClusterWorker', _Initializable, {
|
|
18
|
+
send ({
|
|
19
|
+
message
|
|
20
|
+
}) {
|
|
21
|
+
return new Promise((resolve, reject) => {
|
|
22
|
+
_process.send(message, error => {
|
|
23
|
+
if (error) {
|
|
24
|
+
reject(_Error({
|
|
25
|
+
details: {
|
|
26
|
+
message
|
|
27
|
+
},
|
|
28
|
+
error,
|
|
29
|
+
message: 'Error sending message to primary'
|
|
30
|
+
}));
|
|
31
|
+
} else {
|
|
32
|
+
resolve();
|
|
33
|
+
}
|
|
34
|
+
});
|
|
35
|
+
});
|
|
36
|
+
},
|
|
37
|
+
_destroy (...args) {
|
|
38
|
+
Object.keys(this._processEvents).forEach(eventName => {
|
|
39
|
+
_process.removeListener(eventName, this._processEvents[eventName]);
|
|
40
|
+
});
|
|
41
|
+
|
|
42
|
+
this._processEvents = void null;
|
|
43
|
+
|
|
44
|
+
Reflect.apply(_Initializable.prototype._destroy, this, args);
|
|
45
|
+
},
|
|
46
|
+
_destroyComplete ({
|
|
47
|
+
timeout = 6765
|
|
48
|
+
} = {}) {
|
|
49
|
+
if (_cluster.worker) {
|
|
50
|
+
_cluster.worker.disconnect();
|
|
51
|
+
|
|
52
|
+
_later(timeout, () => {
|
|
53
|
+
_cluster.worker.kill();
|
|
54
|
+
}).unref();
|
|
55
|
+
}
|
|
56
|
+
},
|
|
57
|
+
_eventPrimaryDisconnect () {
|
|
58
|
+
_moduleLogger.info('Primary disconnected');
|
|
59
|
+
},
|
|
60
|
+
_eventPrimaryMessage (event) {
|
|
61
|
+
if (event.data.message && event.data.message.type) {
|
|
62
|
+
const method = this[`_eventPrimaryMessage_${event.data.message.type}`];
|
|
63
|
+
|
|
64
|
+
if (typeof method === 'function') {
|
|
65
|
+
Reflect.apply(method, this, [
|
|
66
|
+
event
|
|
67
|
+
]);
|
|
68
|
+
}
|
|
69
|
+
}
|
|
70
|
+
},
|
|
71
|
+
_init (...args) {
|
|
72
|
+
this._processEvents = {
|
|
73
|
+
disconnect: () => {
|
|
74
|
+
this._publish('primaryDisconnect');
|
|
75
|
+
},
|
|
76
|
+
message: (message, handle) => {
|
|
77
|
+
this._publish('primaryMessage', {
|
|
78
|
+
handle,
|
|
79
|
+
message
|
|
80
|
+
});
|
|
81
|
+
}
|
|
82
|
+
};
|
|
83
|
+
|
|
84
|
+
return Reflect.apply(_Initializable.prototype._init, this, args);
|
|
85
|
+
},
|
|
86
|
+
_initialize () {
|
|
87
|
+
if (_cluster.isWorker) {
|
|
88
|
+
_moduleLogger.info('Initializing worker');
|
|
89
|
+
|
|
90
|
+
Object.keys(this._processEvents).forEach(eventName => {
|
|
91
|
+
_process.on(eventName, this._processEvents[eventName]);
|
|
92
|
+
});
|
|
93
|
+
} else {
|
|
94
|
+
throw _Error({
|
|
95
|
+
message: 'Cluster worker can not initialize in non-worker process'
|
|
96
|
+
});
|
|
97
|
+
}
|
|
98
|
+
},
|
|
99
|
+
_initializeComplete () {
|
|
100
|
+
_process.send('ready');
|
|
101
|
+
}
|
|
102
|
+
}, {
|
|
103
|
+
get worker () {
|
|
104
|
+
return _cluster.worker;
|
|
105
|
+
},
|
|
106
|
+
get workerId () {
|
|
107
|
+
return _workerId;
|
|
108
|
+
},
|
|
109
|
+
_pubsub: {
|
|
110
|
+
primaryDisconnect: {
|
|
111
|
+
completeFunction: '_eventPrimaryDisconnect'
|
|
112
|
+
},
|
|
113
|
+
primaryMessage: {
|
|
114
|
+
completeFunction: '_eventPrimaryMessage'
|
|
115
|
+
}
|
|
116
|
+
}
|
|
117
|
+
});
|
package/package.json
CHANGED
|
@@ -2,42 +2,47 @@
|
|
|
2
2
|
"author": "Steven Olmsted <steven.olmsted@ibigroup.com>",
|
|
3
3
|
"bugs": "https://github.com/ibi-group/isotropic-cluster/issues",
|
|
4
4
|
"dependencies": {
|
|
5
|
-
"isotropic-
|
|
6
|
-
"isotropic-
|
|
7
|
-
"isotropic-
|
|
8
|
-
"isotropic-
|
|
9
|
-
"isotropic-
|
|
5
|
+
"isotropic-backoff": "~0.1.0",
|
|
6
|
+
"isotropic-error": "~0.14.0",
|
|
7
|
+
"isotropic-initializable": "~0.13.0",
|
|
8
|
+
"isotropic-later": "~0.15.0",
|
|
9
|
+
"isotropic-logger": "~0.5.0",
|
|
10
|
+
"isotropic-make": "~0.14.0"
|
|
10
11
|
},
|
|
11
12
|
"description": "A reusable and extendable platform to manage local process clusters",
|
|
12
13
|
"devDependencies": {
|
|
13
|
-
"
|
|
14
|
-
"
|
|
15
|
-
"isotropic-dev-dependencies": "~0.3.1"
|
|
14
|
+
"isotropic-dev-dependencies": "~0.4.0",
|
|
15
|
+
"isotropic-logger-pretty": "~0.5.0"
|
|
16
16
|
},
|
|
17
17
|
"engines": {
|
|
18
|
-
"node": "^
|
|
19
|
-
"npm": "^
|
|
18
|
+
"node": "^26.5.0",
|
|
19
|
+
"npm": "^11.17.0"
|
|
20
20
|
},
|
|
21
21
|
"files": [
|
|
22
22
|
"lib"
|
|
23
23
|
],
|
|
24
24
|
"homepage": "https://github.com/ibi-group/isotropic-cluster",
|
|
25
25
|
"keywords": [
|
|
26
|
-
"
|
|
26
|
+
"cluster",
|
|
27
|
+
"fork",
|
|
28
|
+
"ipc",
|
|
29
|
+
"isotropic",
|
|
30
|
+
"primary",
|
|
31
|
+
"process",
|
|
32
|
+
"restart",
|
|
33
|
+
"worker"
|
|
27
34
|
],
|
|
28
35
|
"license": "BSD-3-Clause",
|
|
29
36
|
"name": "isotropic-cluster",
|
|
30
37
|
"repository": "github:ibi-group/isotropic-cluster",
|
|
31
38
|
"scripts": {
|
|
32
|
-
"
|
|
33
|
-
"
|
|
34
|
-
"
|
|
35
|
-
"
|
|
36
|
-
"prepare": "npm run build",
|
|
37
|
-
"prepublishOnly": "npm test --coverage",
|
|
39
|
+
"lint": "eslint eslint.config.js lib test",
|
|
40
|
+
"posttest": "c8 -c ./node_modules/isotropic-dev-dependencies/config/c8.json check-coverage",
|
|
41
|
+
"prepare": "node ./node_modules/isotropic-dev-dependencies/lib/install-git-hooks.js",
|
|
42
|
+
"prepublishOnly": "npm test",
|
|
38
43
|
"pretest": "npm run lint",
|
|
39
|
-
"test": "
|
|
44
|
+
"test": "c8 -c ./node_modules/isotropic-dev-dependencies/config/c8.json node --test"
|
|
40
45
|
},
|
|
41
46
|
"type": "module",
|
|
42
|
-
"version": "0.
|
|
47
|
+
"version": "0.6.0"
|
|
43
48
|
}
|