aether-hub 1.2.8 → 1.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/commands/claim.js +258 -292
- package/commands/delegations.js +412 -412
- package/commands/emergency.js +607 -667
- package/commands/multisig.js +47 -88
- package/commands/rewards.js +479 -866
- package/commands/stake-positions.js +205 -205
- package/commands/tx-history.js +346 -346
- package/index.js +31 -41
- package/package.json +3 -1
- package/sdk/README.md +210 -0
- package/sdk/index.js +1013 -0
- package/sdk/package.json +33 -0
- package/sdk/rpc.js +108 -0
- package/sdk/test.js +85 -0
- package/theme.js +211 -0
package/commands/emergency.js
CHANGED
|
@@ -1,667 +1,607 @@
|
|
|
1
|
-
#!/usr/bin/env node
|
|
2
|
-
/**
|
|
3
|
-
* aether-cli emergency - Emergency Response & Network Alert System
|
|
4
|
-
*
|
|
5
|
-
*
|
|
6
|
-
*
|
|
7
|
-
*
|
|
8
|
-
*
|
|
9
|
-
*
|
|
10
|
-
* aether emergency
|
|
11
|
-
* aether emergency
|
|
12
|
-
* aether emergency
|
|
13
|
-
* aether emergency
|
|
14
|
-
* aether emergency
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
const
|
|
19
|
-
const
|
|
20
|
-
const
|
|
21
|
-
const
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
const
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
}
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
}
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
function
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
const
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
}
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
const
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
const
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
}
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
|
|
205
|
-
|
|
206
|
-
|
|
207
|
-
|
|
208
|
-
|
|
209
|
-
|
|
210
|
-
|
|
211
|
-
}
|
|
212
|
-
}
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
|
|
216
|
-
|
|
217
|
-
|
|
218
|
-
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
|
|
224
|
-
}
|
|
225
|
-
|
|
226
|
-
|
|
227
|
-
|
|
228
|
-
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
};
|
|
235
|
-
}
|
|
236
|
-
|
|
237
|
-
//
|
|
238
|
-
|
|
239
|
-
|
|
240
|
-
|
|
241
|
-
|
|
242
|
-
|
|
243
|
-
|
|
244
|
-
|
|
245
|
-
if (
|
|
246
|
-
|
|
247
|
-
|
|
248
|
-
|
|
249
|
-
|
|
250
|
-
|
|
251
|
-
}
|
|
252
|
-
|
|
253
|
-
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
//
|
|
257
|
-
|
|
258
|
-
|
|
259
|
-
|
|
260
|
-
|
|
261
|
-
|
|
262
|
-
|
|
263
|
-
|
|
264
|
-
|
|
265
|
-
|
|
266
|
-
|
|
267
|
-
|
|
268
|
-
|
|
269
|
-
|
|
270
|
-
|
|
271
|
-
|
|
272
|
-
|
|
273
|
-
|
|
274
|
-
|
|
275
|
-
|
|
276
|
-
|
|
277
|
-
|
|
278
|
-
|
|
279
|
-
|
|
280
|
-
|
|
281
|
-
}
|
|
282
|
-
|
|
283
|
-
|
|
284
|
-
|
|
285
|
-
|
|
286
|
-
|
|
287
|
-
|
|
288
|
-
|
|
289
|
-
|
|
290
|
-
|
|
291
|
-
|
|
292
|
-
|
|
293
|
-
|
|
294
|
-
|
|
295
|
-
|
|
296
|
-
|
|
297
|
-
|
|
298
|
-
|
|
299
|
-
|
|
300
|
-
|
|
301
|
-
|
|
302
|
-
|
|
303
|
-
|
|
304
|
-
|
|
305
|
-
|
|
306
|
-
|
|
307
|
-
|
|
308
|
-
|
|
309
|
-
|
|
310
|
-
|
|
311
|
-
|
|
312
|
-
|
|
313
|
-
|
|
314
|
-
|
|
315
|
-
|
|
316
|
-
|
|
317
|
-
|
|
318
|
-
|
|
319
|
-
|
|
320
|
-
|
|
321
|
-
|
|
322
|
-
|
|
323
|
-
|
|
324
|
-
|
|
325
|
-
|
|
326
|
-
|
|
327
|
-
|
|
328
|
-
|
|
329
|
-
|
|
330
|
-
|
|
331
|
-
|
|
332
|
-
|
|
333
|
-
|
|
334
|
-
|
|
335
|
-
|
|
336
|
-
|
|
337
|
-
|
|
338
|
-
const
|
|
339
|
-
|
|
340
|
-
|
|
341
|
-
console.log(`\n${C.
|
|
342
|
-
|
|
343
|
-
|
|
344
|
-
|
|
345
|
-
|
|
346
|
-
|
|
347
|
-
|
|
348
|
-
|
|
349
|
-
|
|
350
|
-
|
|
351
|
-
|
|
352
|
-
|
|
353
|
-
|
|
354
|
-
|
|
355
|
-
|
|
356
|
-
|
|
357
|
-
|
|
358
|
-
|
|
359
|
-
|
|
360
|
-
|
|
361
|
-
|
|
362
|
-
|
|
363
|
-
|
|
364
|
-
|
|
365
|
-
|
|
366
|
-
|
|
367
|
-
|
|
368
|
-
|
|
369
|
-
|
|
370
|
-
|
|
371
|
-
|
|
372
|
-
|
|
373
|
-
|
|
374
|
-
|
|
375
|
-
|
|
376
|
-
|
|
377
|
-
|
|
378
|
-
|
|
379
|
-
|
|
380
|
-
|
|
381
|
-
|
|
382
|
-
|
|
383
|
-
|
|
384
|
-
|
|
385
|
-
|
|
386
|
-
|
|
387
|
-
|
|
388
|
-
|
|
389
|
-
|
|
390
|
-
|
|
391
|
-
|
|
392
|
-
|
|
393
|
-
|
|
394
|
-
}
|
|
395
|
-
|
|
396
|
-
|
|
397
|
-
|
|
398
|
-
|
|
399
|
-
|
|
400
|
-
|
|
401
|
-
{
|
|
402
|
-
{
|
|
403
|
-
|
|
404
|
-
|
|
405
|
-
{
|
|
406
|
-
|
|
407
|
-
|
|
408
|
-
|
|
409
|
-
|
|
410
|
-
|
|
411
|
-
|
|
412
|
-
|
|
413
|
-
|
|
414
|
-
|
|
415
|
-
|
|
416
|
-
|
|
417
|
-
|
|
418
|
-
|
|
419
|
-
|
|
420
|
-
|
|
421
|
-
|
|
422
|
-
|
|
423
|
-
|
|
424
|
-
|
|
425
|
-
|
|
426
|
-
|
|
427
|
-
|
|
428
|
-
|
|
429
|
-
|
|
430
|
-
|
|
431
|
-
|
|
432
|
-
|
|
433
|
-
|
|
434
|
-
}
|
|
435
|
-
|
|
436
|
-
|
|
437
|
-
|
|
438
|
-
|
|
439
|
-
|
|
440
|
-
|
|
441
|
-
|
|
442
|
-
|
|
443
|
-
|
|
444
|
-
|
|
445
|
-
|
|
446
|
-
|
|
447
|
-
|
|
448
|
-
|
|
449
|
-
|
|
450
|
-
|
|
451
|
-
|
|
452
|
-
|
|
453
|
-
|
|
454
|
-
|
|
455
|
-
}
|
|
456
|
-
|
|
457
|
-
|
|
458
|
-
|
|
459
|
-
|
|
460
|
-
|
|
461
|
-
}
|
|
462
|
-
|
|
463
|
-
|
|
464
|
-
|
|
465
|
-
|
|
466
|
-
|
|
467
|
-
|
|
468
|
-
|
|
469
|
-
|
|
470
|
-
|
|
471
|
-
|
|
472
|
-
|
|
473
|
-
|
|
474
|
-
|
|
475
|
-
|
|
476
|
-
|
|
477
|
-
|
|
478
|
-
|
|
479
|
-
|
|
480
|
-
|
|
481
|
-
|
|
482
|
-
|
|
483
|
-
|
|
484
|
-
|
|
485
|
-
|
|
486
|
-
|
|
487
|
-
|
|
488
|
-
|
|
489
|
-
|
|
490
|
-
|
|
491
|
-
|
|
492
|
-
} else {
|
|
493
|
-
console.log(` ${C.yellow}⚠
|
|
494
|
-
|
|
495
|
-
}
|
|
496
|
-
}
|
|
497
|
-
console.log(` ${C.
|
|
498
|
-
|
|
499
|
-
|
|
500
|
-
|
|
501
|
-
|
|
502
|
-
|
|
503
|
-
|
|
504
|
-
|
|
505
|
-
|
|
506
|
-
|
|
507
|
-
|
|
508
|
-
|
|
509
|
-
|
|
510
|
-
|
|
511
|
-
|
|
512
|
-
|
|
513
|
-
|
|
514
|
-
|
|
515
|
-
console.log(
|
|
516
|
-
|
|
517
|
-
|
|
518
|
-
|
|
519
|
-
|
|
520
|
-
console.log(` ${C.
|
|
521
|
-
|
|
522
|
-
|
|
523
|
-
|
|
524
|
-
|
|
525
|
-
|
|
526
|
-
|
|
527
|
-
|
|
528
|
-
|
|
529
|
-
|
|
530
|
-
|
|
531
|
-
|
|
532
|
-
|
|
533
|
-
|
|
534
|
-
|
|
535
|
-
|
|
536
|
-
|
|
537
|
-
|
|
538
|
-
|
|
539
|
-
|
|
540
|
-
|
|
541
|
-
|
|
542
|
-
|
|
543
|
-
|
|
544
|
-
|
|
545
|
-
|
|
546
|
-
|
|
547
|
-
|
|
548
|
-
|
|
549
|
-
|
|
550
|
-
|
|
551
|
-
|
|
552
|
-
|
|
553
|
-
|
|
554
|
-
|
|
555
|
-
|
|
556
|
-
|
|
557
|
-
|
|
558
|
-
|
|
559
|
-
|
|
560
|
-
|
|
561
|
-
|
|
562
|
-
|
|
563
|
-
|
|
564
|
-
|
|
565
|
-
|
|
566
|
-
|
|
567
|
-
|
|
568
|
-
|
|
569
|
-
|
|
570
|
-
|
|
571
|
-
|
|
572
|
-
|
|
573
|
-
|
|
574
|
-
|
|
575
|
-
|
|
576
|
-
|
|
577
|
-
|
|
578
|
-
|
|
579
|
-
|
|
580
|
-
|
|
581
|
-
|
|
582
|
-
|
|
583
|
-
}
|
|
584
|
-
|
|
585
|
-
|
|
586
|
-
|
|
587
|
-
|
|
588
|
-
|
|
589
|
-
|
|
590
|
-
|
|
591
|
-
|
|
592
|
-
}
|
|
593
|
-
|
|
594
|
-
|
|
595
|
-
|
|
596
|
-
|
|
597
|
-
|
|
598
|
-
|
|
599
|
-
|
|
600
|
-
|
|
601
|
-
|
|
602
|
-
|
|
603
|
-
|
|
604
|
-
|
|
605
|
-
|
|
606
|
-
|
|
607
|
-
|
|
608
|
-
};
|
|
609
|
-
|
|
610
|
-
const msgIndex = allArgs.findIndex(a => a === '--message');
|
|
611
|
-
if (msgIndex !== -1 && allArgs[msgIndex + 1]) opts.message = allArgs[msgIndex + 1];
|
|
612
|
-
|
|
613
|
-
const intIndex = allArgs.findIndex(a => a === '--interval');
|
|
614
|
-
if (intIndex !== -1 && allArgs[intIndex + 1]) opts.interval = parseInt(allArgs[intIndex + 1], 10);
|
|
615
|
-
|
|
616
|
-
const linesIndex = allArgs.findIndex(a => a === '--lines');
|
|
617
|
-
if (linesIndex !== -1 && allArgs[linesIndex + 1]) opts.lines = parseInt(allArgs[linesIndex + 1], 10);
|
|
618
|
-
|
|
619
|
-
switch (subcmd) {
|
|
620
|
-
case 'status':
|
|
621
|
-
await emergencyStatus(opts);
|
|
622
|
-
break;
|
|
623
|
-
case 'monitor':
|
|
624
|
-
await emergencyMonitor(opts);
|
|
625
|
-
break;
|
|
626
|
-
case 'check':
|
|
627
|
-
await emergencyCheck(opts);
|
|
628
|
-
break;
|
|
629
|
-
case 'alert':
|
|
630
|
-
await emergencyAlert(opts);
|
|
631
|
-
break;
|
|
632
|
-
case 'failover':
|
|
633
|
-
await emergencyFailover(opts);
|
|
634
|
-
break;
|
|
635
|
-
case 'history':
|
|
636
|
-
await emergencyHistory(opts);
|
|
637
|
-
break;
|
|
638
|
-
default:
|
|
639
|
-
console.log(`\n${C.bright}${C.cyan}aether emergency${C.reset} — Emergency Response & Alert System\n`);
|
|
640
|
-
console.log(`Usage: ${C.cyan}aether emergency <command>${C.reset}\n`);
|
|
641
|
-
console.log(`Commands:`);
|
|
642
|
-
console.log(` ${C.cyan}status${C.reset} Check current emergency level (default)`);
|
|
643
|
-
console.log(` ${C.cyan}monitor${C.reset} Continuous monitoring loop (--interval <sec>)`);
|
|
644
|
-
console.log(` ${C.cyan}check${C.reset} Run full diagnostic checks`);
|
|
645
|
-
console.log(` ${C.cyan}alert${C.reset} Issue a governance alert (--message "...")`);
|
|
646
|
-
console.log(` ${C.cyan}failover${C.reset} Trigger backup node failover`);
|
|
647
|
-
console.log(` ${C.cyan}history${C.reset} Show recent emergency events (--lines <n>)`);
|
|
648
|
-
console.log(`\nOptions:`);
|
|
649
|
-
console.log(` --rpc <url> RPC endpoint (default: $AETHER_RPC or localhost)`);
|
|
650
|
-
console.log(` --json JSON output\n`);
|
|
651
|
-
}
|
|
652
|
-
}
|
|
653
|
-
|
|
654
|
-
// Exported emergency command handler for CLI integration
|
|
655
|
-
async function emergencyCommand() {
|
|
656
|
-
return main();
|
|
657
|
-
}
|
|
658
|
-
|
|
659
|
-
// Run if called directly
|
|
660
|
-
if (require.main === module) {
|
|
661
|
-
main().catch(err => {
|
|
662
|
-
console.error(`\n${C.red}Error in emergency command:${C.reset}`, err.message, '\n');
|
|
663
|
-
process.exit(1);
|
|
664
|
-
});
|
|
665
|
-
}
|
|
666
|
-
|
|
667
|
-
module.exports = { emergencyCommand };
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
/**
|
|
3
|
+
* aether-cli emergency - Emergency Response & Network Alert System
|
|
4
|
+
*
|
|
5
|
+
* Uses @jellylegsai/aether-sdk for REAL blockchain RPC calls.
|
|
6
|
+
* Detects network emergencies (halts, consensus failures, high tps drops),
|
|
7
|
+
* monitors validator liveness, issues governance alerts, and triggers backups.
|
|
8
|
+
*
|
|
9
|
+
* Usage:
|
|
10
|
+
* aether emergency status # Check current emergency level
|
|
11
|
+
* aether emergency monitor [--interval 30] # Continuous monitoring loop
|
|
12
|
+
* aether emergency alert --message "..." # Issue a governance alert
|
|
13
|
+
* aether emergency failover # Trigger backup node failover
|
|
14
|
+
* aether emergency history # Show recent emergency events
|
|
15
|
+
* aether emergency check # Run all diagnostics
|
|
16
|
+
*/
|
|
17
|
+
|
|
18
|
+
const fs = require('fs');
|
|
19
|
+
const os = require('os');
|
|
20
|
+
const path = require('path');
|
|
21
|
+
const readline = require('readline');
|
|
22
|
+
|
|
23
|
+
// Import SDK for real RPC calls
|
|
24
|
+
const sdkPath = path.join(__dirname, '..', 'sdk', 'index.js');
|
|
25
|
+
const { AetherClient } = require(sdkPath);
|
|
26
|
+
|
|
27
|
+
// Import theme
|
|
28
|
+
const theme = require('../theme');
|
|
29
|
+
const { C, ICONS, BANNERS } = theme;
|
|
30
|
+
|
|
31
|
+
const DEFAULT_RPC = process.env.AETHER_RPC || 'http://127.0.0.1:8899';
|
|
32
|
+
const EMERGENCY_LOG = path.join(os.homedir(), '.aether', 'emergency.log');
|
|
33
|
+
|
|
34
|
+
// ---------------------------------------------------------------------------
|
|
35
|
+
// Paths
|
|
36
|
+
// ---------------------------------------------------------------------------
|
|
37
|
+
|
|
38
|
+
function getAetherDir() {
|
|
39
|
+
return path.join(os.homedir(), '.aether');
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
function getValidatorConfig() {
|
|
43
|
+
const p = path.join(getAetherDir(), 'validator-identity.json');
|
|
44
|
+
if (!fs.existsSync(p)) return null;
|
|
45
|
+
try {
|
|
46
|
+
return JSON.parse(fs.readFileSync(p, 'utf8'));
|
|
47
|
+
} catch {
|
|
48
|
+
return null;
|
|
49
|
+
}
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
// ---------------------------------------------------------------------------
|
|
53
|
+
// SDK helpers
|
|
54
|
+
// ---------------------------------------------------------------------------
|
|
55
|
+
|
|
56
|
+
function createClient(rpcUrl) {
|
|
57
|
+
return new AetherClient({ rpcUrl });
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
/** Check if node is responding using SDK */
|
|
61
|
+
async function checkNodeHealth(rpc) {
|
|
62
|
+
const client = createClient(rpc);
|
|
63
|
+
try {
|
|
64
|
+
const slot = await client.getSlot();
|
|
65
|
+
return { ok: true, slot };
|
|
66
|
+
} catch (err) {
|
|
67
|
+
return { ok: false, error: err.message };
|
|
68
|
+
}
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
/** Check slot progression using SDK */
|
|
72
|
+
async function checkSlotProgression(rpc, count = 3) {
|
|
73
|
+
const client = createClient(rpc);
|
|
74
|
+
const slots = [];
|
|
75
|
+
for (let i = 0; i < count; i++) {
|
|
76
|
+
try {
|
|
77
|
+
const slot = await client.getSlot();
|
|
78
|
+
slots.push(slot);
|
|
79
|
+
if (i < count - 1) await new Promise(r => setTimeout(r, 2000));
|
|
80
|
+
} catch {
|
|
81
|
+
slots.push(null);
|
|
82
|
+
}
|
|
83
|
+
}
|
|
84
|
+
const valid = slots.filter(s => s !== null);
|
|
85
|
+
if (valid.length < 2) return { halted: true, slots };
|
|
86
|
+
const advancing = valid[valid.length - 1] > valid[0];
|
|
87
|
+
return { halted: !advancing, slots, delta: valid.length > 1 ? valid[valid.length - 1] - valid[0] : 0 };
|
|
88
|
+
}
|
|
89
|
+
|
|
90
|
+
/** Check block height using SDK */
|
|
91
|
+
async function checkBlockHeight(rpc) {
|
|
92
|
+
const client = createClient(rpc);
|
|
93
|
+
try {
|
|
94
|
+
const height = await client.getBlockHeight();
|
|
95
|
+
return { ok: true, blockHeight: height };
|
|
96
|
+
} catch (err) {
|
|
97
|
+
return { ok: false, error: err.message };
|
|
98
|
+
}
|
|
99
|
+
}
|
|
100
|
+
|
|
101
|
+
/** Check epoch info using SDK */
|
|
102
|
+
async function checkEpoch(rpc) {
|
|
103
|
+
const client = createClient(rpc);
|
|
104
|
+
try {
|
|
105
|
+
return await client.getEpochInfo();
|
|
106
|
+
} catch {
|
|
107
|
+
return null;
|
|
108
|
+
}
|
|
109
|
+
}
|
|
110
|
+
|
|
111
|
+
/** Check TPS using SDK */
|
|
112
|
+
async function checkTPS(rpc) {
|
|
113
|
+
const client = createClient(rpc);
|
|
114
|
+
try {
|
|
115
|
+
return await client.getTPS();
|
|
116
|
+
} catch {
|
|
117
|
+
return null;
|
|
118
|
+
}
|
|
119
|
+
}
|
|
120
|
+
|
|
121
|
+
/** Check peers using SDK */
|
|
122
|
+
async function checkPeers(rpc) {
|
|
123
|
+
const client = createClient(rpc);
|
|
124
|
+
try {
|
|
125
|
+
const validators = await client.getValidators();
|
|
126
|
+
return Array.isArray(validators) ? validators.length : 0;
|
|
127
|
+
} catch {
|
|
128
|
+
return null;
|
|
129
|
+
}
|
|
130
|
+
}
|
|
131
|
+
|
|
132
|
+
/** Check local validator status */
|
|
133
|
+
async function checkValidatorStatus() {
|
|
134
|
+
const identity = getValidatorConfig();
|
|
135
|
+
if (!identity) return { configured: false };
|
|
136
|
+
return {
|
|
137
|
+
configured: true,
|
|
138
|
+
identity: identity.identity ?? identity.nodeId ?? 'unknown',
|
|
139
|
+
stake: identity.stake ?? identity.delegated ?? null,
|
|
140
|
+
};
|
|
141
|
+
}
|
|
142
|
+
|
|
143
|
+
/** Issue alert via SDK */
|
|
144
|
+
async function submitAlert(rpc, message, validator) {
|
|
145
|
+
const client = createClient(rpc);
|
|
146
|
+
try {
|
|
147
|
+
// Try to submit to governance endpoint
|
|
148
|
+
return await client._httpPost('/v1/governance/alert', {
|
|
149
|
+
message,
|
|
150
|
+
validator: validator?.identity ?? 'unknown',
|
|
151
|
+
timestamp: new Date().toISOString(),
|
|
152
|
+
});
|
|
153
|
+
} catch (err) {
|
|
154
|
+
return { success: false, error: err.message };
|
|
155
|
+
}
|
|
156
|
+
}
|
|
157
|
+
|
|
158
|
+
// ---------------------------------------------------------------------------
|
|
159
|
+
// Emergency log
|
|
160
|
+
// ---------------------------------------------------------------------------
|
|
161
|
+
|
|
162
|
+
function logEmergency(level, message, details = {}) {
|
|
163
|
+
const entry = {
|
|
164
|
+
timestamp: new Date().toISOString(),
|
|
165
|
+
level,
|
|
166
|
+
message,
|
|
167
|
+
details,
|
|
168
|
+
};
|
|
169
|
+
const line = JSON.stringify(entry);
|
|
170
|
+
const dir = path.dirname(EMERGENCY_LOG);
|
|
171
|
+
if (!fs.existsSync(dir)) fs.mkdirSync(dir, { recursive: true });
|
|
172
|
+
fs.appendFileSync(EMERGENCY_LOG, line + '\n');
|
|
173
|
+
return entry;
|
|
174
|
+
}
|
|
175
|
+
|
|
176
|
+
function readEmergencyLog(lines = 50) {
|
|
177
|
+
if (!fs.existsSync(EMERGENCY_LOG)) return [];
|
|
178
|
+
const content = fs.readFileSync(EMERGENCY_LOG, 'utf8');
|
|
179
|
+
const all = content.split('\n').filter(Boolean).map(l => {
|
|
180
|
+
try { return JSON.parse(l); }
|
|
181
|
+
catch { return null; }
|
|
182
|
+
}).filter(Boolean);
|
|
183
|
+
return all.slice(-lines);
|
|
184
|
+
}
|
|
185
|
+
|
|
186
|
+
// ---------------------------------------------------------------------------
|
|
187
|
+
// Emergency level assessment
|
|
188
|
+
// ---------------------------------------------------------------------------
|
|
189
|
+
|
|
190
|
+
function assessEmergencyLevel(results) {
|
|
191
|
+
let level = 0; // 0=ok, 1=warning, 2=elevated, 3=critical
|
|
192
|
+
|
|
193
|
+
if (!results.nodeHealth.ok) level = Math.max(level, 3);
|
|
194
|
+
if (results.slotHalt.halted) level = Math.max(level, 3);
|
|
195
|
+
if (results.lowTps !== null && results.lowTps < 10) level = Math.max(level, 2);
|
|
196
|
+
if (results.lowPeers !== null && results.lowPeers < 3) level = Math.max(level, 1);
|
|
197
|
+
if (!results.epoch || !results.epoch.epoch) level = Math.max(level, 1);
|
|
198
|
+
|
|
199
|
+
return level;
|
|
200
|
+
}
|
|
201
|
+
|
|
202
|
+
const LEVEL_LABELS = ['OK', 'WARNING', 'ELEVATED', 'CRITICAL'];
|
|
203
|
+
const LEVEL_COLORS = [C.green, C.yellow, C.magenta, C.red];
|
|
204
|
+
|
|
205
|
+
// ---------------------------------------------------------------------------
|
|
206
|
+
// Commands
|
|
207
|
+
// ---------------------------------------------------------------------------
|
|
208
|
+
|
|
209
|
+
async function emergencyStatus(opts) {
|
|
210
|
+
const { rpc, json } = opts;
|
|
211
|
+
console.log(`\n${BANNERS.emergency}\n`);
|
|
212
|
+
console.log(` ${C.dim}RPC:${C.reset} ${rpc}\n`);
|
|
213
|
+
|
|
214
|
+
const [nodeHealth, slotHalt, blockHeight, epoch, tps, peers, validator] = await Promise.all([
|
|
215
|
+
checkNodeHealth(rpc),
|
|
216
|
+
checkSlotProgression(rpc, 3),
|
|
217
|
+
checkBlockHeight(rpc),
|
|
218
|
+
checkEpoch(rpc),
|
|
219
|
+
checkTPS(rpc),
|
|
220
|
+
checkPeers(rpc),
|
|
221
|
+
checkValidatorStatus(),
|
|
222
|
+
]);
|
|
223
|
+
|
|
224
|
+
const results = { nodeHealth, slotHalt, blockHeight, epoch, tps, peers, validator };
|
|
225
|
+
const level = assessEmergencyLevel(results);
|
|
226
|
+
|
|
227
|
+
if (json) {
|
|
228
|
+
console.log(JSON.stringify({ ...results, emergencyLevel: level }, null, 2));
|
|
229
|
+
return;
|
|
230
|
+
}
|
|
231
|
+
|
|
232
|
+
// Node health
|
|
233
|
+
const healthIcon = nodeHealth.ok ? `${ICONS.success}` : `${ICONS.error}`;
|
|
234
|
+
const healthLabel = nodeHealth.ok ? `Slot ${nodeHealth.slot}` : nodeHealth.error;
|
|
235
|
+
console.log(` ${healthIcon} ${C.bright}Node Health${C.reset} ${healthLabel}`);
|
|
236
|
+
|
|
237
|
+
// Slot progression
|
|
238
|
+
const haltIcon = slotHalt.halted ? `${ICONS.warning} HALTED` : `${ICONS.success} Advancing`;
|
|
239
|
+
const haltLabel = slotHalt.halted
|
|
240
|
+
? `No new slots in ${slotHalt.slots.length} checks`
|
|
241
|
+
: `+${slotHalt.delta} slots over ${slotHalt.slots.length} checks`;
|
|
242
|
+
console.log(` ${haltIcon} ${C.bright}Slot Progress${C.reset} ${C.dim}${haltLabel}${C.reset}`);
|
|
243
|
+
|
|
244
|
+
// Block height
|
|
245
|
+
if (blockHeight.ok) {
|
|
246
|
+
console.log(` ${ICONS.success} ${C.bright}Block Height${C.reset} ${blockHeight.blockHeight}`);
|
|
247
|
+
}
|
|
248
|
+
|
|
249
|
+
// Epoch
|
|
250
|
+
if (epoch && epoch.epoch) {
|
|
251
|
+
console.log(` ${ICONS.success} ${C.bright}Epoch${C.reset} ${epoch.epoch} ${C.dim}(progress: ${epoch.slot_index ?? '?'}/${epoch.slots_in_epoch ?? '?'})${C.reset}`);
|
|
252
|
+
} else {
|
|
253
|
+
console.log(` ${ICONS.warning} ${C.bright}Epoch${C.reset} ${C.dim}unavailable${C.reset}`);
|
|
254
|
+
}
|
|
255
|
+
|
|
256
|
+
// TPS
|
|
257
|
+
const tpsColor = tps === null ? C.yellow : (tps < 10 ? C.red : C.green);
|
|
258
|
+
const tpsIcon = tps === null ? '?' : (tps < 10 ? ICONS.warning : ICONS.success);
|
|
259
|
+
console.log(` ${tpsColor}${tpsIcon}${C.reset} ${C.bright}TPS${C.reset} ${tps !== null ? tps.toFixed(1) : C.dim + 'unavailable' + C.reset}`);
|
|
260
|
+
|
|
261
|
+
// Peers
|
|
262
|
+
const peerColor = peers === null ? C.yellow : (peers < 3 ? C.red : C.green);
|
|
263
|
+
const peerIcon = peers === null ? '?' : (peers < 3 ? ICONS.warning : ICONS.success);
|
|
264
|
+
console.log(` ${peerColor}${peerIcon}${C.reset} ${C.bright}Connected Peers${C.reset} ${peers !== null ? peers : C.dim + 'unavailable' + C.reset}`);
|
|
265
|
+
|
|
266
|
+
// Validator
|
|
267
|
+
if (validator.configured) {
|
|
268
|
+
console.log(` ${C.cyan}▸${C.reset} ${C.bright}Validator${C.reset} ${validator.identity.substring(0, 16)}... ${C.dim}stake: ${validator.stake ?? '?'}${C.reset}`);
|
|
269
|
+
} else {
|
|
270
|
+
console.log(` ${C.dim}▸ Validator${C.reset} ${C.dim}not configured${C.reset}`);
|
|
271
|
+
}
|
|
272
|
+
|
|
273
|
+
// Emergency level banner
|
|
274
|
+
console.log(`\n ${C.bright}Emergency Level:${C.reset} ${LEVEL_COLORS[level]}${LEVEL_LABELS[level]}${C.reset}\n`);
|
|
275
|
+
|
|
276
|
+
if (level >= 2) {
|
|
277
|
+
console.log(` ${ICONS.warning} Run:${C.reset} ${C.cyan}aether emergency monitor${C.reset} to watch continuously`);
|
|
278
|
+
console.log(` ${ICONS.warning} Run:${C.reset} ${C.cyan}aether emergency check${C.reset} for full diagnostics\n`);
|
|
279
|
+
}
|
|
280
|
+
|
|
281
|
+
logEmergency(LEVEL_LABELS[level], 'Status check', { level, results: { slot: nodeHealth.slot, halted: slotHalt.halted, tps, peers } });
|
|
282
|
+
|
|
283
|
+
if (level >= 2 && !json) console.log(` ${C.dim}Logged to:${C.reset} ${EMERGENCY_LOG}\n`);
|
|
284
|
+
}
|
|
285
|
+
|
|
286
|
+
async function emergencyMonitor(opts) {
|
|
287
|
+
const { rpc, json, interval = 30 } = opts;
|
|
288
|
+
const rl = readline.createInterface({ input: process.stdin, output: process.stdout });
|
|
289
|
+
|
|
290
|
+
console.log(`\n${C.bright}${C.red}🔴 Aether Emergency Monitor${C.reset}`);
|
|
291
|
+
console.log(` Monitoring every ${interval}s. ${C.dim}Press Ctrl+C to stop.${C.reset}\n`);
|
|
292
|
+
|
|
293
|
+
let lastLevel = 0;
|
|
294
|
+
let count = 0;
|
|
295
|
+
|
|
296
|
+
const doCheck = async () => {
|
|
297
|
+
count++;
|
|
298
|
+
const [nodeHealth, slotHalt, blockHeight, epoch, tps, peers, validator] = await Promise.all([
|
|
299
|
+
checkNodeHealth(rpc),
|
|
300
|
+
checkSlotProgression(rpc, 3),
|
|
301
|
+
checkBlockHeight(rpc),
|
|
302
|
+
checkEpoch(rpc),
|
|
303
|
+
checkTPS(rpc),
|
|
304
|
+
checkPeers(rpc),
|
|
305
|
+
checkValidatorStatus(),
|
|
306
|
+
]);
|
|
307
|
+
|
|
308
|
+
const results = { nodeHealth, slotHalt, blockHeight, epoch, tps, peers, validator };
|
|
309
|
+
const level = assessEmergencyLevel(results);
|
|
310
|
+
const ts = new Date().toISOString().substring(11, 19);
|
|
311
|
+
|
|
312
|
+
if (json) {
|
|
313
|
+
console.log(JSON.stringify({ ts, ...results, emergencyLevel: level }));
|
|
314
|
+
} else {
|
|
315
|
+
const icon = level === 0 ? `${C.green}✓` : level === 1 ? `${C.yellow}${ICONS.warning}` : level === 2 ? `${C.magenta}${ICONS.lightning}` : `${C.red}🔴`;
|
|
316
|
+
const slot = nodeHealth.ok ? `slot=${nodeHealth.slot}` : 'DOWN';
|
|
317
|
+
const halt = slotHalt.halted ? 'HALT!' : `+${slotHalt.delta}`;
|
|
318
|
+
const tpsStr = tps !== null ? `tps=${tps.toFixed(0)}` : 'tps=?';
|
|
319
|
+
const peerStr = peers !== null ? `peers=${peers}` : '';
|
|
320
|
+
console.log(`${icon} [${ts}] ${slot} ${halt} ${tpsStr} ${peerStr} | Level: ${LEVEL_COLORS[level]}${LEVEL_LABELS[level]}${C.reset}`);
|
|
321
|
+
}
|
|
322
|
+
|
|
323
|
+
if (level > lastLevel) {
|
|
324
|
+
logEmergency(LEVEL_LABELS[level], 'Escalation', { from: lastLevel, to: level });
|
|
325
|
+
lastLevel = level;
|
|
326
|
+
}
|
|
327
|
+
|
|
328
|
+
if (level >= 3) {
|
|
329
|
+
logEmergency('CRITICAL', 'Network emergency - CRITICAL level', results);
|
|
330
|
+
}
|
|
331
|
+
};
|
|
332
|
+
|
|
333
|
+
// First run
|
|
334
|
+
await doCheck();
|
|
335
|
+
|
|
336
|
+
// Repeat
|
|
337
|
+
const intervalMs = interval * 1000;
|
|
338
|
+
const id = setInterval(doCheck, intervalMs);
|
|
339
|
+
|
|
340
|
+
// Handle Ctrl+C
|
|
341
|
+
const cleanup = () => { clearInterval(id); rl.close(); console.log(`\n${C.dim}Monitor stopped after ${count} checks.${C.reset}\n`); };
|
|
342
|
+
process.on('SIGINT', cleanup);
|
|
343
|
+
}
|
|
344
|
+
|
|
345
|
+
async function emergencyCheck(opts) {
|
|
346
|
+
const { rpc, json } = opts;
|
|
347
|
+
console.log(`\n${C.bright}${C.cyan}🔬 Aether Emergency Diagnostics${C.reset}\n`);
|
|
348
|
+
|
|
349
|
+
const checks = [
|
|
350
|
+
{ name: 'Node Health', fn: () => checkNodeHealth(rpc) },
|
|
351
|
+
{ name: 'Slot Progression', fn: () => checkSlotProgression(rpc, 5) },
|
|
352
|
+
{ name: 'Block Height', fn: () => checkBlockHeight(rpc) },
|
|
353
|
+
{ name: 'Epoch Info', fn: () => checkEpoch(rpc) },
|
|
354
|
+
{ name: 'TPS', fn: () => checkTPS(rpc) },
|
|
355
|
+
{ name: 'Peers', fn: () => checkPeers(rpc) },
|
|
356
|
+
{ name: 'Validator Config', fn: () => checkValidatorStatus() },
|
|
357
|
+
];
|
|
358
|
+
|
|
359
|
+
const results = {};
|
|
360
|
+
let pass = 0, fail = 0, warn = 0;
|
|
361
|
+
|
|
362
|
+
for (const check of checks) {
|
|
363
|
+
process.stdout.write(` ${C.dim}Checking ${check.name}...${C.reset} `);
|
|
364
|
+
const result = await check.fn();
|
|
365
|
+
results[check.name] = result;
|
|
366
|
+
|
|
367
|
+
const ok = result && (result.ok === undefined ? true : result.ok);
|
|
368
|
+
if (ok !== false && check.name !== 'Validator Config') {
|
|
369
|
+
// Determine status by check type
|
|
370
|
+
let status = `${C.green}✓ PASS${C.reset}`;
|
|
371
|
+
if (check.name === 'Slot Progression' && result.halted) {
|
|
372
|
+
status = `${C.red}✗ FAIL${C.reset}`; fail++;
|
|
373
|
+
} else if (check.name === 'TPS' && result !== null && result < 10) {
|
|
374
|
+
status = `${C.yellow}⚠ WARN${C.reset}`; warn++;
|
|
375
|
+
} else if (check.name === 'Peers' && result !== null && result < 3) {
|
|
376
|
+
status = `${C.yellow}⚠ WARN${C.reset}`; warn++;
|
|
377
|
+
} else if (check.name === 'Node Health' && !result.ok) {
|
|
378
|
+
status = `${C.red}✗ FAIL${C.reset}`; fail++;
|
|
379
|
+
} else {
|
|
380
|
+
pass++;
|
|
381
|
+
}
|
|
382
|
+
console.log(status);
|
|
383
|
+
} else if (check.name === 'Validator Config') {
|
|
384
|
+
if (result.configured) {
|
|
385
|
+
console.log(`${C.green}✓ CONFIGURED${C.reset}`);
|
|
386
|
+
pass++;
|
|
387
|
+
} else {
|
|
388
|
+
console.log(`${C.yellow}⚠ NOT CONFIGURED${C.reset}`);
|
|
389
|
+
warn++;
|
|
390
|
+
}
|
|
391
|
+
} else {
|
|
392
|
+
console.log(`${C.red}✗ FAIL${C.reset}`);
|
|
393
|
+
fail++;
|
|
394
|
+
}
|
|
395
|
+
}
|
|
396
|
+
|
|
397
|
+
console.log(`\n ${C.bright}Results:${C.reset} ${C.green}${pass} pass${C.reset} ${C.yellow}${warn} warn${C.reset} ${C.red}${fail} fail${C.reset}\n`);
|
|
398
|
+
|
|
399
|
+
if (fail > 0) {
|
|
400
|
+
console.log(` ${C.red}⚠ Network emergency detected!${C.reset}`);
|
|
401
|
+
console.log(` ${C.dim} Run:${C.reset} ${C.cyan}aether emergency monitor${C.reset} to watch continuously`);
|
|
402
|
+
console.log(` ${C.dim} Run:${C.reset} ${C.cyan}aether emergency failover${C.reset} to trigger backup\n`);
|
|
403
|
+
logEmergency('CRITICAL', 'Diagnostics failed', { pass, fail, warn });
|
|
404
|
+
} else if (warn > 0) {
|
|
405
|
+
console.log(` ${C.yellow}⚠ Some metrics are degraded but network is operational.${C.reset}\n`);
|
|
406
|
+
logEmergency('WARNING', 'Diagnostics warning', { pass, fail, warn });
|
|
407
|
+
} else {
|
|
408
|
+
console.log(` ${C.green}✓ All checks passed. Network is healthy.${C.reset}\n`);
|
|
409
|
+
logEmergency('OK', 'Diagnostics passed', { pass, fail, warn });
|
|
410
|
+
}
|
|
411
|
+
|
|
412
|
+
if (json) {
|
|
413
|
+
console.log(JSON.stringify({ results, pass, fail, warn }, null, 2));
|
|
414
|
+
}
|
|
415
|
+
}
|
|
416
|
+
|
|
417
|
+
async function emergencyAlert(opts) {
|
|
418
|
+
const { message, rpc } = opts;
|
|
419
|
+
if (!message) {
|
|
420
|
+
console.log(`\n${C.red}Error: --message is required${C.reset}`);
|
|
421
|
+
console.log(` ${C.dim}Usage: aether emergency alert --message "Network alert text"${C.reset}\n`);
|
|
422
|
+
return;
|
|
423
|
+
}
|
|
424
|
+
|
|
425
|
+
console.log(`\n${C.bright}🔶 Issuing Governance Alert${C.reset}\n`);
|
|
426
|
+
console.log(` ${C.dim}Message:${C.reset} ${message}\n`);
|
|
427
|
+
|
|
428
|
+
// Try to submit alert to governance endpoint
|
|
429
|
+
const identity = getValidatorConfig();
|
|
430
|
+
const result = await submitAlert(rpc, message, identity);
|
|
431
|
+
|
|
432
|
+
if (result.success || result.alert_id) {
|
|
433
|
+
console.log(` ${C.green}✓ Alert issued successfully${C.reset}`);
|
|
434
|
+
console.log(` ${C.dim}Alert ID: ${result.alert_id ?? 'unknown'}${C.reset}\n`);
|
|
435
|
+
logEmergency('ELEVATED', `Alert issued: ${message}`, { alertId: result.alert_id });
|
|
436
|
+
} else {
|
|
437
|
+
console.log(` ${C.yellow}⚠ Alert submitted (check response):${C.reset}`);
|
|
438
|
+
console.log(` ${JSON.stringify(result)}\n`);
|
|
439
|
+
logEmergency('ELEVATED', `Local alert (network unreachable): ${message}`, { error: result.error });
|
|
440
|
+
}
|
|
441
|
+
}
|
|
442
|
+
|
|
443
|
+
async function emergencyFailover(opts) {
|
|
444
|
+
const { rpc, json } = opts;
|
|
445
|
+
console.log(`\n${C.bright}${C.magenta}⚡ Aether Emergency Failover${C.reset}\n`);
|
|
446
|
+
|
|
447
|
+
const identity = getValidatorConfig();
|
|
448
|
+
if (!identity) {
|
|
449
|
+
console.log(` ${C.red}✗ No validator identity found.${C.reset}`);
|
|
450
|
+
console.log(` ${C.dim} Run:${C.reset} ${C.cyan}aether init${C.reset} first to configure validator\n`);
|
|
451
|
+
return;
|
|
452
|
+
}
|
|
453
|
+
|
|
454
|
+
console.log(` ${C.dim}Validator:${C.reset} ${identity.identity ?? identity.nodeId ?? 'unknown'}`);
|
|
455
|
+
console.log(` ${C.dim}Checking backup node status...${C.reset}\n`);
|
|
456
|
+
|
|
457
|
+
// Check if backup RPC is configured
|
|
458
|
+
const backupRpc = process.env.AETHER_BACKUP_RPC;
|
|
459
|
+
if (!backupRpc) {
|
|
460
|
+
console.log(` ${C.yellow}⚠ AETHER_BACKUP_RPC not set.${C.reset}`);
|
|
461
|
+
console.log(` ${C.dim} Export it in your environment to enable automatic failover.${C.reset}`);
|
|
462
|
+
console.log(` ${C.dim} export AETHER_BACKUP_RPC=http://backup-node:8899${C.reset}\n`);
|
|
463
|
+
}
|
|
464
|
+
|
|
465
|
+
// Check current node
|
|
466
|
+
const [health, slotHalt] = await Promise.all([
|
|
467
|
+
checkNodeHealth(rpc),
|
|
468
|
+
checkSlotProgression(rpc, 3),
|
|
469
|
+
]);
|
|
470
|
+
|
|
471
|
+
console.log(` ${C.dim}Current node health:${C.reset} ${health.ok ? `${C.green}✓${C.reset}` : `${C.red}✗${C.reset}`}`);
|
|
472
|
+
console.log(` ${C.dim}Slot status:${C.reset} ${slotHalt.halted ? `${C.red}HALTED${C.reset}` : `${C.green}Advancing${C.reset}`}\n`);
|
|
473
|
+
|
|
474
|
+
if (!health.ok || slotHalt.halted) {
|
|
475
|
+
console.log(` ${C.red}⚠ Primary node is down! Attempting failover...${C.reset}\n`);
|
|
476
|
+
|
|
477
|
+
if (backupRpc) {
|
|
478
|
+
try {
|
|
479
|
+
const backupHealth = await checkNodeHealth(backupRpc);
|
|
480
|
+
if (backupHealth.ok) {
|
|
481
|
+
console.log(` ${C.green}✓ Backup node is healthy!${C.reset}`);
|
|
482
|
+
console.log(` ${C.green}✓ Failover would succeed.${C.reset}`);
|
|
483
|
+
console.log(` ${C.dim} Update your AETHER_RPC to:${C.reset} ${backupRpc}\n`);
|
|
484
|
+
logEmergency('CRITICAL', 'Failover needed and backup available', { backupRpc });
|
|
485
|
+
} else {
|
|
486
|
+
console.log(` ${C.red}✗ Backup node is also unreachable.${C.reset}\n`);
|
|
487
|
+
logEmergency('CRITICAL', 'Failover failed - both primary and backup down', {});
|
|
488
|
+
}
|
|
489
|
+
} catch {
|
|
490
|
+
console.log(` ${C.red}✗ Backup node check failed.${C.reset}\n`);
|
|
491
|
+
}
|
|
492
|
+
} else {
|
|
493
|
+
console.log(` ${C.yellow}⚠ Set AETHER_BACKUP_RPC to enable automatic failover.${C.reset}\n`);
|
|
494
|
+
logEmergency('CRITICAL', 'Failover needed but no backup configured', {});
|
|
495
|
+
}
|
|
496
|
+
} else {
|
|
497
|
+
console.log(` ${C.green}✓ Primary node is healthy. No failover needed.${C.reset}\n`);
|
|
498
|
+
}
|
|
499
|
+
}
|
|
500
|
+
|
|
501
|
+
async function emergencyHistory(opts) {
|
|
502
|
+
const { json, lines = 20 } = opts;
|
|
503
|
+
const events = readEmergencyLog(lines);
|
|
504
|
+
|
|
505
|
+
if (events.length === 0) {
|
|
506
|
+
console.log(`\n${C.dim}No emergency events logged.${C.reset}\n`);
|
|
507
|
+
return;
|
|
508
|
+
}
|
|
509
|
+
|
|
510
|
+
if (json) {
|
|
511
|
+
console.log(JSON.stringify(events, null, 2));
|
|
512
|
+
return;
|
|
513
|
+
}
|
|
514
|
+
|
|
515
|
+
console.log(`\n${C.bright}📋 Recent Emergency Events${C.reset} ${C.dim}(last ${events.length})${C.reset}\n`);
|
|
516
|
+
|
|
517
|
+
for (const ev of events) {
|
|
518
|
+
const levelColor = ev.level === 'OK' ? C.green : ev.level === 'WARNING' ? C.yellow : ev.level === 'ELEVATED' ? C.magenta : C.red;
|
|
519
|
+
const ts = ev.timestamp ? ev.timestamp.substring(0, 19) : '?';
|
|
520
|
+
console.log(` ${levelColor}[${ev.level}]${C.reset} ${C.dim}${ts}${C.reset} — ${ev.message}`);
|
|
521
|
+
}
|
|
522
|
+
console.log();
|
|
523
|
+
}
|
|
524
|
+
|
|
525
|
+
// ---------------------------------------------------------------------------
|
|
526
|
+
// CLI argument parsing
|
|
527
|
+
// ---------------------------------------------------------------------------
|
|
528
|
+
|
|
529
|
+
function parseArgs() {
|
|
530
|
+
const args = process.argv.slice(3); // [node, index.js, emergency, <subcmd>, ...]
|
|
531
|
+
return args;
|
|
532
|
+
}
|
|
533
|
+
|
|
534
|
+
async function main() {
|
|
535
|
+
const rawArgs = parseArgs();
|
|
536
|
+
const subcmd = rawArgs[0] || 'status';
|
|
537
|
+
|
|
538
|
+
const allArgs = rawArgs.slice(1);
|
|
539
|
+
const rpcIndex = allArgs.findIndex(a => a === '--rpc');
|
|
540
|
+
const rpc = rpcIndex !== -1 && allArgs[rpcIndex + 1] ? allArgs[rpcIndex + 1] : DEFAULT_RPC;
|
|
541
|
+
|
|
542
|
+
const opts = {
|
|
543
|
+
rpc,
|
|
544
|
+
json: allArgs.includes('--json'),
|
|
545
|
+
message: null,
|
|
546
|
+
interval: 30,
|
|
547
|
+
lines: 20,
|
|
548
|
+
};
|
|
549
|
+
|
|
550
|
+
const msgIndex = allArgs.findIndex(a => a === '--message');
|
|
551
|
+
if (msgIndex !== -1 && allArgs[msgIndex + 1]) opts.message = allArgs[msgIndex + 1];
|
|
552
|
+
|
|
553
|
+
const intIndex = allArgs.findIndex(a => a === '--interval');
|
|
554
|
+
if (intIndex !== -1 && allArgs[intIndex + 1]) opts.interval = parseInt(allArgs[intIndex + 1], 10);
|
|
555
|
+
|
|
556
|
+
const linesIndex = allArgs.findIndex(a => a === '--lines');
|
|
557
|
+
if (linesIndex !== -1 && allArgs[linesIndex + 1]) opts.lines = parseInt(allArgs[linesIndex + 1], 10);
|
|
558
|
+
|
|
559
|
+
switch (subcmd) {
|
|
560
|
+
case 'status':
|
|
561
|
+
await emergencyStatus(opts);
|
|
562
|
+
break;
|
|
563
|
+
case 'monitor':
|
|
564
|
+
await emergencyMonitor(opts);
|
|
565
|
+
break;
|
|
566
|
+
case 'check':
|
|
567
|
+
await emergencyCheck(opts);
|
|
568
|
+
break;
|
|
569
|
+
case 'alert':
|
|
570
|
+
await emergencyAlert(opts);
|
|
571
|
+
break;
|
|
572
|
+
case 'failover':
|
|
573
|
+
await emergencyFailover(opts);
|
|
574
|
+
break;
|
|
575
|
+
case 'history':
|
|
576
|
+
await emergencyHistory(opts);
|
|
577
|
+
break;
|
|
578
|
+
default:
|
|
579
|
+
console.log(`\n${C.bright}${C.cyan}aether emergency${C.reset} — Emergency Response & Alert System\n`);
|
|
580
|
+
console.log(`Usage: ${C.cyan}aether emergency <command>${C.reset}\n`);
|
|
581
|
+
console.log(`Commands:`);
|
|
582
|
+
console.log(` ${C.cyan}status${C.reset} Check current emergency level (default)`);
|
|
583
|
+
console.log(` ${C.cyan}monitor${C.reset} Continuous monitoring loop (--interval <sec>)`);
|
|
584
|
+
console.log(` ${C.cyan}check${C.reset} Run full diagnostic checks`);
|
|
585
|
+
console.log(` ${C.cyan}alert${C.reset} Issue a governance alert (--message "...")`);
|
|
586
|
+
console.log(` ${C.cyan}failover${C.reset} Trigger backup node failover`);
|
|
587
|
+
console.log(` ${C.cyan}history${C.reset} Show recent emergency events (--lines <n>)`);
|
|
588
|
+
console.log(`\nOptions:`);
|
|
589
|
+
console.log(` --rpc <url> RPC endpoint (default: $AETHER_RPC or localhost)`);
|
|
590
|
+
console.log(` --json JSON output\n`);
|
|
591
|
+
}
|
|
592
|
+
}
|
|
593
|
+
|
|
594
|
+
// Exported emergency command handler for CLI integration
|
|
595
|
+
async function emergencyCommand() {
|
|
596
|
+
return main();
|
|
597
|
+
}
|
|
598
|
+
|
|
599
|
+
// Run if called directly
|
|
600
|
+
if (require.main === module) {
|
|
601
|
+
main().catch(err => {
|
|
602
|
+
console.error(`\n${C.red}Error in emergency command:${C.reset}`, err.message, '\n');
|
|
603
|
+
process.exit(1);
|
|
604
|
+
});
|
|
605
|
+
}
|
|
606
|
+
|
|
607
|
+
module.exports = { emergencyCommand };
|