aether-hub 1.1.1 → 1.1.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/commands/emergency.js +657 -0
- package/commands/rewards.js +600 -600
- package/commands/wallet.js +168 -0
- package/index.js +18 -0
- package/package.json +1 -1
|
@@ -0,0 +1,657 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
/**
|
|
3
|
+
* aether-cli emergency - Emergency Response & Network Alert System
|
|
4
|
+
*
|
|
5
|
+
* Detects network emergencies (halts, consensus failures, high tps drops),
|
|
6
|
+
* monitors validator liveness, issues governance alerts, and triggers backups.
|
|
7
|
+
*
|
|
8
|
+
* Usage:
|
|
9
|
+
* aether emergency status # Check current emergency level
|
|
10
|
+
* aether emergency monitor [--interval 30] # Continuous monitoring loop
|
|
11
|
+
* aether emergency alert --message "..." # Issue a governance alert
|
|
12
|
+
* aether emergency failover # Trigger backup node failover
|
|
13
|
+
* aether emergency history # Show recent emergency events
|
|
14
|
+
* aether emergency check # Run all diagnostics
|
|
15
|
+
*/
|
|
16
|
+
|
|
17
|
+
const http = require('http');
|
|
18
|
+
const https = require('https');
|
|
19
|
+
const fs = require('fs');
|
|
20
|
+
const os = require('os');
|
|
21
|
+
const path = require('path');
|
|
22
|
+
const readline = require('readline');
|
|
23
|
+
|
|
24
|
+
// ANSI colours
|
|
25
|
+
const C = {
|
|
26
|
+
reset: '\x1b[0m',
|
|
27
|
+
bright: '\x1b[1m',
|
|
28
|
+
dim: '\x1b[2m',
|
|
29
|
+
red: '\x1b[31m',
|
|
30
|
+
green: '\x1b[32m',
|
|
31
|
+
yellow: '\x1b[33m',
|
|
32
|
+
blue: '\x1b[34m',
|
|
33
|
+
cyan: '\x1b[36m',
|
|
34
|
+
magenta: '\x1b[35m',
|
|
35
|
+
white: '\x1b[37m',
|
|
36
|
+
};
|
|
37
|
+
|
|
38
|
+
const DEFAULT_RPC = process.env.AETHER_RPC || 'http://127.0.0.1:8899';
|
|
39
|
+
const EMERGENCY_LOG = path.join(os.homedir(), '.aether', 'emergency.log');
|
|
40
|
+
|
|
41
|
+
// ---------------------------------------------------------------------------
|
|
42
|
+
// Paths
|
|
43
|
+
// ---------------------------------------------------------------------------
|
|
44
|
+
|
|
45
|
+
function getAetherDir() {
|
|
46
|
+
return path.join(os.homedir(), '.aether');
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
function getValidatorConfig() {
|
|
50
|
+
const p = path.join(getAetherDir(), 'validator-identity.json');
|
|
51
|
+
if (!fs.existsSync(p)) return null;
|
|
52
|
+
try {
|
|
53
|
+
return JSON.parse(fs.readFileSync(p, 'utf8'));
|
|
54
|
+
} catch {
|
|
55
|
+
return null;
|
|
56
|
+
}
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
// ---------------------------------------------------------------------------
|
|
60
|
+
// HTTP helpers
|
|
61
|
+
// ---------------------------------------------------------------------------
|
|
62
|
+
|
|
63
|
+
function httpRequest(rpcUrl, pathStr, method = 'GET') {
|
|
64
|
+
return new Promise((resolve, reject) => {
|
|
65
|
+
const url = new URL(pathStr, rpcUrl);
|
|
66
|
+
const isHttps = url.protocol === 'https:';
|
|
67
|
+
const lib = isHttps ? https : http;
|
|
68
|
+
|
|
69
|
+
const req = lib.request({
|
|
70
|
+
hostname: url.hostname,
|
|
71
|
+
port: url.port || (isHttps ? 443 : 80),
|
|
72
|
+
path: url.pathname + url.search,
|
|
73
|
+
method,
|
|
74
|
+
timeout: 5000,
|
|
75
|
+
headers: { 'Content-Type': 'application/json' },
|
|
76
|
+
}, (res) => {
|
|
77
|
+
let data = '';
|
|
78
|
+
res.on('data', (chunk) => (data += chunk));
|
|
79
|
+
res.on('end', () => {
|
|
80
|
+
try { resolve(JSON.parse(data)); }
|
|
81
|
+
catch { resolve({ raw: data }); }
|
|
82
|
+
});
|
|
83
|
+
});
|
|
84
|
+
|
|
85
|
+
req.on('error', reject);
|
|
86
|
+
req.on('timeout', () => { req.destroy(); reject(new Error('Request timeout')); });
|
|
87
|
+
req.end();
|
|
88
|
+
});
|
|
89
|
+
}
|
|
90
|
+
|
|
91
|
+
function httpPost(rpcUrl, pathStr, body) {
|
|
92
|
+
return new Promise((resolve, reject) => {
|
|
93
|
+
const url = new URL(pathStr, rpcUrl);
|
|
94
|
+
const isHttps = url.protocol === 'https:';
|
|
95
|
+
const lib = isHttps ? https : http;
|
|
96
|
+
const bodyStr = JSON.stringify(body);
|
|
97
|
+
|
|
98
|
+
const req = lib.request({
|
|
99
|
+
hostname: url.hostname,
|
|
100
|
+
port: url.port || (isHttps ? 443 : 80),
|
|
101
|
+
path: url.pathname + url.search,
|
|
102
|
+
method: 'POST',
|
|
103
|
+
timeout: 5000,
|
|
104
|
+
headers: {
|
|
105
|
+
'Content-Type': 'application/json',
|
|
106
|
+
'Content-Length': Buffer.byteLength(bodyStr),
|
|
107
|
+
},
|
|
108
|
+
}, (res) => {
|
|
109
|
+
let data = '';
|
|
110
|
+
res.on('data', (chunk) => (data += chunk));
|
|
111
|
+
res.on('end', () => {
|
|
112
|
+
try { resolve(JSON.parse(data)); }
|
|
113
|
+
catch { resolve(data); }
|
|
114
|
+
});
|
|
115
|
+
});
|
|
116
|
+
|
|
117
|
+
req.on('error', reject);
|
|
118
|
+
req.on('timeout', () => { req.destroy(); reject(new Error('Request timeout')); });
|
|
119
|
+
req.write(bodyStr);
|
|
120
|
+
req.end();
|
|
121
|
+
});
|
|
122
|
+
}
|
|
123
|
+
|
|
124
|
+
// ---------------------------------------------------------------------------
|
|
125
|
+
// Emergency log
|
|
126
|
+
// ---------------------------------------------------------------------------
|
|
127
|
+
|
|
128
|
+
function logEmergency(level, message, details = {}) {
|
|
129
|
+
const entry = {
|
|
130
|
+
timestamp: new Date().toISOString(),
|
|
131
|
+
level,
|
|
132
|
+
message,
|
|
133
|
+
details,
|
|
134
|
+
};
|
|
135
|
+
const line = JSON.stringify(entry);
|
|
136
|
+
const dir = path.dirname(EMERGENCY_LOG);
|
|
137
|
+
if (!fs.existsSync(dir)) fs.mkdirSync(dir, { recursive: true });
|
|
138
|
+
fs.appendFileSync(EMERGENCY_LOG, line + '\n');
|
|
139
|
+
return entry;
|
|
140
|
+
}
|
|
141
|
+
|
|
142
|
+
function readEmergencyLog(lines = 50) {
|
|
143
|
+
if (!fs.existsSync(EMERGENCY_LOG)) return [];
|
|
144
|
+
const content = fs.readFileSync(EMERGENCY_LOG, 'utf8');
|
|
145
|
+
const all = content.split('\n').filter(Boolean).map(l => {
|
|
146
|
+
try { return JSON.parse(l); }
|
|
147
|
+
catch { return null; }
|
|
148
|
+
}).filter(Boolean);
|
|
149
|
+
return all.slice(-lines);
|
|
150
|
+
}
|
|
151
|
+
|
|
152
|
+
// ---------------------------------------------------------------------------
|
|
153
|
+
// Diagnostic checks
|
|
154
|
+
// ---------------------------------------------------------------------------
|
|
155
|
+
|
|
156
|
+
/** Check if node is responding */
|
|
157
|
+
async function checkNodeHealth(rpc) {
|
|
158
|
+
try {
|
|
159
|
+
const res = await httpRequest(rpc, '/v1/slot');
|
|
160
|
+
return { ok: true, slot: res.slot ?? res.root_slot ?? null };
|
|
161
|
+
} catch (err) {
|
|
162
|
+
return { ok: false, error: err.message };
|
|
163
|
+
}
|
|
164
|
+
}
|
|
165
|
+
|
|
166
|
+
/** Check slot progression (is network advancing?) */
|
|
167
|
+
async function checkSlotProgression(rpc, count = 3) {
|
|
168
|
+
const slots = [];
|
|
169
|
+
for (let i = 0; i < count; i++) {
|
|
170
|
+
try {
|
|
171
|
+
const res = await httpRequest(rpc, '/v1/slot');
|
|
172
|
+
slots.push(res.slot ?? res.root_slot ?? null);
|
|
173
|
+
if (i < count - 1) await new Promise(r => setTimeout(r, 2000));
|
|
174
|
+
} catch {
|
|
175
|
+
slots.push(null);
|
|
176
|
+
}
|
|
177
|
+
}
|
|
178
|
+
const valid = slots.filter(s => s !== null);
|
|
179
|
+
if (valid.length < 2) return { halted: true, slots };
|
|
180
|
+
const advancing = valid[valid.length - 1] > valid[0];
|
|
181
|
+
return { halted: !advancing, slots, delta: valid.length > 1 ? valid[valid.length - 1] - valid[0] : 0 };
|
|
182
|
+
}
|
|
183
|
+
|
|
184
|
+
/** Check block height consistency */
|
|
185
|
+
async function checkBlockHeight(rpc) {
|
|
186
|
+
try {
|
|
187
|
+
const res = await httpRequest(rpc, '/v1/block_height');
|
|
188
|
+
return { ok: true, blockHeight: res.block_height ?? null };
|
|
189
|
+
} catch (err) {
|
|
190
|
+
return { ok: false, error: err.message };
|
|
191
|
+
}
|
|
192
|
+
}
|
|
193
|
+
|
|
194
|
+
/** Check epoch info */
|
|
195
|
+
async function checkEpoch(rpc) {
|
|
196
|
+
try {
|
|
197
|
+
const res = await httpRequest(rpc, '/v1/epoch');
|
|
198
|
+
return res;
|
|
199
|
+
} catch {
|
|
200
|
+
return null;
|
|
201
|
+
}
|
|
202
|
+
}
|
|
203
|
+
|
|
204
|
+
/** Check TPS for dramatic drops */
|
|
205
|
+
async function checkTPS(rpc) {
|
|
206
|
+
try {
|
|
207
|
+
const res = await httpRequest(rpc, '/v1/tps');
|
|
208
|
+
return res.tps ?? res.tps_avg ?? res.transactions_per_second ?? null;
|
|
209
|
+
} catch {
|
|
210
|
+
return null;
|
|
211
|
+
}
|
|
212
|
+
}
|
|
213
|
+
|
|
214
|
+
/** Check connected peers count */
|
|
215
|
+
async function checkPeers(rpc) {
|
|
216
|
+
try {
|
|
217
|
+
const res = await httpRequest(rpc, '/v1/validators');
|
|
218
|
+
if (Array.isArray(res.validators)) return res.validators.length;
|
|
219
|
+
if (Array.isArray(res)) return res.length;
|
|
220
|
+
return null;
|
|
221
|
+
} catch {
|
|
222
|
+
return null;
|
|
223
|
+
}
|
|
224
|
+
}
|
|
225
|
+
|
|
226
|
+
/** Check local validator status */
|
|
227
|
+
async function checkValidatorStatus() {
|
|
228
|
+
const identity = getValidatorConfig();
|
|
229
|
+
if (!identity) return { configured: false };
|
|
230
|
+
return {
|
|
231
|
+
configured: true,
|
|
232
|
+
identity: identity.identity ?? identity.nodeId ?? 'unknown',
|
|
233
|
+
stake: identity.stake ?? identity.delegated ?? null,
|
|
234
|
+
};
|
|
235
|
+
}
|
|
236
|
+
|
|
237
|
+
// ---------------------------------------------------------------------------
|
|
238
|
+
// Emergency level assessment
|
|
239
|
+
// ---------------------------------------------------------------------------
|
|
240
|
+
|
|
241
|
+
function assessEmergencyLevel(results) {
|
|
242
|
+
let level = 0; // 0=ok, 1=warning, 2=elevated, 3=critical
|
|
243
|
+
|
|
244
|
+
if (!results.nodeHealth.ok) level = Math.max(level, 3);
|
|
245
|
+
if (results.slotHalt.halted) level = Math.max(level, 3);
|
|
246
|
+
if (results.lowTps !== null && results.lowTps < 10) level = Math.max(level, 2);
|
|
247
|
+
if (results.lowPeers !== null && results.lowPeers < 3) level = Math.max(level, 1);
|
|
248
|
+
if (!results.epoch || !results.epoch.epoch) level = Math.max(level, 1);
|
|
249
|
+
|
|
250
|
+
return level;
|
|
251
|
+
}
|
|
252
|
+
|
|
253
|
+
const LEVEL_LABELS = ['OK', 'WARNING', 'ELEVATED', 'CRITICAL'];
|
|
254
|
+
const LEVEL_COLORS = [C.green, C.yellow, C.magenta, C.red];
|
|
255
|
+
|
|
256
|
+
// ---------------------------------------------------------------------------
|
|
257
|
+
// Commands
|
|
258
|
+
// ---------------------------------------------------------------------------
|
|
259
|
+
|
|
260
|
+
async function emergencyStatus(opts) {
|
|
261
|
+
const { rpc, json } = opts;
|
|
262
|
+
console.log(`\n${C.bright}${C.cyan}🔔 Aether Emergency Status${C.reset}\n`);
|
|
263
|
+
console.log(` ${C.dim}RPC:${C.reset} ${rpc}\n`);
|
|
264
|
+
|
|
265
|
+
const [nodeHealth, slotHalt, blockHeight, epoch, tps, peers, validator] = await Promise.all([
|
|
266
|
+
checkNodeHealth(rpc),
|
|
267
|
+
checkSlotProgression(rpc, 3),
|
|
268
|
+
checkBlockHeight(rpc),
|
|
269
|
+
checkEpoch(rpc),
|
|
270
|
+
checkTPS(rpc),
|
|
271
|
+
checkPeers(rpc),
|
|
272
|
+
checkValidatorStatus(),
|
|
273
|
+
]);
|
|
274
|
+
|
|
275
|
+
const results = { nodeHealth, slotHalt, blockHeight, epoch, tps, peers, validator };
|
|
276
|
+
const level = assessEmergencyLevel(results);
|
|
277
|
+
|
|
278
|
+
if (json) {
|
|
279
|
+
console.log(JSON.stringify({ ...results, emergencyLevel: level }, null, 2));
|
|
280
|
+
return;
|
|
281
|
+
}
|
|
282
|
+
|
|
283
|
+
// Node health
|
|
284
|
+
const healthIcon = nodeHealth.ok ? `${C.green}✓` : `${C.red}✗`;
|
|
285
|
+
const healthLabel = nodeHealth.ok ? `Slot ${nodeHealth.slot}` : nodeHealth.error;
|
|
286
|
+
console.log(` ${healthIcon} ${C.bright}Node Health${C.reset} ${healthLabel}`);
|
|
287
|
+
|
|
288
|
+
// Slot progression
|
|
289
|
+
const haltIcon = slotHalt.halted ? `${C.red}⚠ HALTED` : `${C.green}✓ Advancing`;
|
|
290
|
+
const haltLabel = slotHalt.halted
|
|
291
|
+
? `No new slots in ${slotHalt.slots.length} checks`
|
|
292
|
+
: `+${slotHalt.delta} slots over ${slotHalt.slots.length} checks`;
|
|
293
|
+
console.log(` ${haltIcon} ${C.bright}Slot Progress${C.reset} ${C.dim}${haltLabel}${C.reset}`);
|
|
294
|
+
|
|
295
|
+
// Block height
|
|
296
|
+
if (blockHeight.ok) {
|
|
297
|
+
console.log(` ${C.green}✓${C.reset} ${C.bright}Block Height${C.reset} ${blockHeight.blockHeight}`);
|
|
298
|
+
}
|
|
299
|
+
|
|
300
|
+
// Epoch
|
|
301
|
+
if (epoch && epoch.epoch) {
|
|
302
|
+
console.log(` ${C.green}✓${C.reset} ${C.bright}Epoch${C.reset} ${epoch.epoch} ${C.dim}(progress: ${epoch.slot_index ?? '?'}/${epoch.slots_in_epoch ?? '?'})${C.reset}`);
|
|
303
|
+
} else {
|
|
304
|
+
console.log(` ${C.yellow}?${C.reset} ${C.bright}Epoch${C.reset} ${C.dim}unavailable${C.reset}`);
|
|
305
|
+
}
|
|
306
|
+
|
|
307
|
+
// TPS
|
|
308
|
+
const tpsColor = tps === null ? C.yellow : (tps < 10 ? C.red : C.green);
|
|
309
|
+
const tpsIcon = tps === null ? '?' : (tps < 10 ? '⚠' : '✓');
|
|
310
|
+
console.log(` ${tpsColor}${tpsIcon}${C.reset} ${C.bright}TPS${C.reset} ${tps !== null ? tps.toFixed(1) : C.dim + 'unavailable' + C.reset}`);
|
|
311
|
+
|
|
312
|
+
// Peers
|
|
313
|
+
const peerColor = peers === null ? C.yellow : (peers < 3 ? C.red : C.green);
|
|
314
|
+
const peerIcon = peers === null ? '?' : (peers < 3 ? '⚠' : '✓');
|
|
315
|
+
console.log(` ${peerColor}${peerIcon}${C.reset} ${C.bright}Connected Peers${C.reset} ${peers !== null ? peers : C.dim + 'unavailable' + C.reset}`);
|
|
316
|
+
|
|
317
|
+
// Validator
|
|
318
|
+
if (validator.configured) {
|
|
319
|
+
console.log(` ${C.cyan}▸${C.reset} ${C.bright}Validator${C.reset} ${validator.identity.substring(0, 16)}... ${C.dim}stake: ${validator.stake ?? '?'}${C.reset}`);
|
|
320
|
+
} else {
|
|
321
|
+
console.log(` ${C.dim}▸ Validator${C.reset} ${C.dim}not configured${C.reset}`);
|
|
322
|
+
}
|
|
323
|
+
|
|
324
|
+
// Emergency level banner
|
|
325
|
+
console.log(`\n ${C.bright}Emergency Level:${C.reset} ${LEVEL_COLORS[level]}${LEVEL_LABELS[level]}${C.reset}\n`);
|
|
326
|
+
|
|
327
|
+
if (level >= 2) {
|
|
328
|
+
console.log(` ${C.yellow}⚠ Run:${C.reset} ${C.cyan}aether emergency monitor${C.reset} to watch continuously`);
|
|
329
|
+
console.log(` ${C.yellow}⚠ Run:${C.reset} ${C.cyan}aether emergency check${C.reset} for full diagnostics\n`);
|
|
330
|
+
}
|
|
331
|
+
|
|
332
|
+
logEmergency(LEVEL_LABELS[level], 'Status check', { level, results: { slot: nodeHealth.slot, halted: slotHalt.halted, tps, peers } });
|
|
333
|
+
|
|
334
|
+
if (level >= 2 && !json) console.log(` ${C.dim}Logged to:${C.reset} ${EMERGENCY_LOG}\n`);
|
|
335
|
+
}
|
|
336
|
+
|
|
337
|
+
async function emergencyMonitor(opts) {
|
|
338
|
+
const { rpc, json, interval = 30 } = opts;
|
|
339
|
+
const rl = readline.createInterface({ input: process.stdin, output: process.stdout });
|
|
340
|
+
|
|
341
|
+
console.log(`\n${C.bright}${C.red}🔴 Aether Emergency Monitor${C.reset}`);
|
|
342
|
+
console.log(` Monitoring every ${interval}s. ${C.dim}Press Ctrl+C to stop.${C.reset}\n`);
|
|
343
|
+
|
|
344
|
+
let lastLevel = 0;
|
|
345
|
+
let count = 0;
|
|
346
|
+
|
|
347
|
+
const doCheck = async () => {
|
|
348
|
+
count++;
|
|
349
|
+
const [nodeHealth, slotHalt, blockHeight, epoch, tps, peers, validator] = await Promise.all([
|
|
350
|
+
checkNodeHealth(rpc),
|
|
351
|
+
checkSlotProgression(rpc, 3),
|
|
352
|
+
checkBlockHeight(rpc),
|
|
353
|
+
checkEpoch(rpc),
|
|
354
|
+
checkTPS(rpc),
|
|
355
|
+
checkPeers(rpc),
|
|
356
|
+
checkValidatorStatus(),
|
|
357
|
+
]);
|
|
358
|
+
|
|
359
|
+
const results = { nodeHealth, slotHalt, blockHeight, epoch, tps, peers, validator };
|
|
360
|
+
const level = assessEmergencyLevel(results);
|
|
361
|
+
const ts = new Date().toISOString().substring(11, 19);
|
|
362
|
+
|
|
363
|
+
if (json) {
|
|
364
|
+
console.log(JSON.stringify({ ts, ...results, emergencyLevel: level }));
|
|
365
|
+
} else {
|
|
366
|
+
const icon = level === 0 ? `${C.green}✓` : level === 1 ? `${C.yellow}⚠` : level === 2 ? `${C.magenta}⚡` : `${C.red}🔴`;
|
|
367
|
+
const slot = nodeHealth.ok ? `slot=${nodeHealth.slot}` : 'DOWN';
|
|
368
|
+
const halt = slotHalt.halted ? 'HALT!' : `+${slotHalt.delta}`;
|
|
369
|
+
const tpsStr = tps !== null ? `tps=${tps.toFixed(0)}` : 'tps=?';
|
|
370
|
+
const peerStr = peers !== null ? `peers=${peers}` : '';
|
|
371
|
+
console.log(`${icon} [${ts}] ${slot} ${halt} ${tpsStr} ${peerStr} | Level: ${LEVEL_COLORS[level]}${LEVEL_LABELS[level]}${C.reset}`);
|
|
372
|
+
}
|
|
373
|
+
|
|
374
|
+
if (level > lastLevel) {
|
|
375
|
+
logEmergency(LEVEL_LABELS[level], 'Escalation', { from: lastLevel, to: level });
|
|
376
|
+
lastLevel = level;
|
|
377
|
+
}
|
|
378
|
+
|
|
379
|
+
if (level >= 3) {
|
|
380
|
+
logEmergency('CRITICAL', 'Network emergency - CRITICAL level', results);
|
|
381
|
+
}
|
|
382
|
+
};
|
|
383
|
+
|
|
384
|
+
// First run
|
|
385
|
+
await doCheck();
|
|
386
|
+
|
|
387
|
+
// Repeat
|
|
388
|
+
const intervalMs = interval * 1000;
|
|
389
|
+
const id = setInterval(doCheck, intervalMs);
|
|
390
|
+
|
|
391
|
+
// Handle Ctrl+C
|
|
392
|
+
const cleanup = () => { clearInterval(id); rl.close(); console.log(`\n${C.dim}Monitor stopped after ${count} checks.${C.reset}\n`); };
|
|
393
|
+
process.on('SIGINT', cleanup);
|
|
394
|
+
}
|
|
395
|
+
|
|
396
|
+
async function emergencyCheck(opts) {
|
|
397
|
+
const { rpc, json } = opts;
|
|
398
|
+
console.log(`\n${C.bright}${C.cyan}🔬 Aether Emergency Diagnostics${C.reset}\n`);
|
|
399
|
+
|
|
400
|
+
const checks = [
|
|
401
|
+
{ name: 'Node Health', fn: () => checkNodeHealth(rpc) },
|
|
402
|
+
{ name: 'Slot Progression', fn: () => checkSlotProgression(rpc, 5) },
|
|
403
|
+
{ name: 'Block Height', fn: () => checkBlockHeight(rpc) },
|
|
404
|
+
{ name: 'Epoch Info', fn: () => checkEpoch(rpc) },
|
|
405
|
+
{ name: 'TPS', fn: () => checkTPS(rpc) },
|
|
406
|
+
{ name: 'Peers', fn: () => checkPeers(rpc) },
|
|
407
|
+
{ name: 'Validator Config', fn: () => checkValidatorStatus() },
|
|
408
|
+
];
|
|
409
|
+
|
|
410
|
+
const results = {};
|
|
411
|
+
let pass = 0, fail = 0, warn = 0;
|
|
412
|
+
|
|
413
|
+
for (const check of checks) {
|
|
414
|
+
process.stdout.write(` ${C.dim}Checking ${check.name}...${C.reset} `);
|
|
415
|
+
const result = await check.fn();
|
|
416
|
+
results[check.name] = result;
|
|
417
|
+
|
|
418
|
+
const ok = result && (result.ok === undefined ? true : result.ok);
|
|
419
|
+
if (ok !== false && check.name !== 'Validator Config') {
|
|
420
|
+
// Determine status by check type
|
|
421
|
+
let status = `${C.green}✓ PASS${C.reset}`;
|
|
422
|
+
if (check.name === 'Slot Progression' && result.halted) {
|
|
423
|
+
status = `${C.red}✗ FAIL${C.reset}`; fail++;
|
|
424
|
+
} else if (check.name === 'TPS' && result !== null && result < 10) {
|
|
425
|
+
status = `${C.yellow}⚠ WARN${C.reset}`; warn++;
|
|
426
|
+
} else if (check.name === 'Peers' && result !== null && result < 3) {
|
|
427
|
+
status = `${C.yellow}⚠ WARN${C.reset}`; warn++;
|
|
428
|
+
} else if (check.name === 'Node Health' && !result.ok) {
|
|
429
|
+
status = `${C.red}✗ FAIL${C.reset}`; fail++;
|
|
430
|
+
} else {
|
|
431
|
+
pass++;
|
|
432
|
+
}
|
|
433
|
+
console.log(status);
|
|
434
|
+
} else if (check.name === 'Validator Config') {
|
|
435
|
+
if (result.configured) {
|
|
436
|
+
console.log(`${C.green}✓ CONFIGURED${C.reset}`);
|
|
437
|
+
pass++;
|
|
438
|
+
} else {
|
|
439
|
+
console.log(`${C.yellow}⚠ NOT CONFIGURED${C.reset}`);
|
|
440
|
+
warn++;
|
|
441
|
+
}
|
|
442
|
+
} else {
|
|
443
|
+
console.log(`${C.red}✗ FAIL${C.reset}`);
|
|
444
|
+
fail++;
|
|
445
|
+
}
|
|
446
|
+
}
|
|
447
|
+
|
|
448
|
+
console.log(`\n ${C.bright}Results:${C.reset} ${C.green}${pass} pass${C.reset} ${C.yellow}${warn} warn${C.reset} ${C.red}${fail} fail${C.reset}\n`);
|
|
449
|
+
|
|
450
|
+
if (fail > 0) {
|
|
451
|
+
console.log(` ${C.red}⚠ Network emergency detected!${C.reset}`);
|
|
452
|
+
console.log(` ${C.dim} Run:${C.reset} ${C.cyan}aether emergency monitor${C.reset} to watch continuously`);
|
|
453
|
+
console.log(` ${C.dim} Run:${C.reset} ${C.cyan}aether emergency failover${C.reset} to trigger backup\n`);
|
|
454
|
+
logEmergency('CRITICAL', 'Diagnostics failed', { pass, fail, warn });
|
|
455
|
+
} else if (warn > 0) {
|
|
456
|
+
console.log(` ${C.yellow}⚠ Some metrics are degraded but network is operational.${C.reset}\n`);
|
|
457
|
+
logEmergency('WARNING', 'Diagnostics warning', { pass, fail, warn });
|
|
458
|
+
} else {
|
|
459
|
+
console.log(` ${C.green}✓ All checks passed. Network is healthy.${C.reset}\n`);
|
|
460
|
+
logEmergency('OK', 'Diagnostics passed', { pass, fail, warn });
|
|
461
|
+
}
|
|
462
|
+
|
|
463
|
+
if (json) {
|
|
464
|
+
console.log(JSON.stringify({ results, pass, fail, warn }, null, 2));
|
|
465
|
+
}
|
|
466
|
+
}
|
|
467
|
+
|
|
468
|
+
async function emergencyAlert(opts) {
|
|
469
|
+
const { message, rpc } = opts;
|
|
470
|
+
if (!message) {
|
|
471
|
+
console.log(`\n${C.red}Error: --message is required${C.reset}`);
|
|
472
|
+
console.log(` ${C.dim}Usage: aether emergency alert --message "Network alert text"${C.reset}\n`);
|
|
473
|
+
return;
|
|
474
|
+
}
|
|
475
|
+
|
|
476
|
+
console.log(`\n${C.bright}🔶 Issuing Governance Alert${C.reset}\n`);
|
|
477
|
+
console.log(` ${C.dim}Message:${C.reset} ${message}\n`);
|
|
478
|
+
|
|
479
|
+
// Try to submit alert to governance endpoint
|
|
480
|
+
try {
|
|
481
|
+
const identity = getValidatorConfig();
|
|
482
|
+
const result = await httpPost(rpc, '/v1/governance/alert', {
|
|
483
|
+
message,
|
|
484
|
+
validator: identity?.identity ?? 'unknown',
|
|
485
|
+
timestamp: new Date().toISOString(),
|
|
486
|
+
});
|
|
487
|
+
|
|
488
|
+
if (result.success || result.alert_id) {
|
|
489
|
+
console.log(` ${C.green}✓ Alert issued successfully${C.reset}`);
|
|
490
|
+
console.log(` ${C.dim}Alert ID: ${result.alert_id ?? 'unknown'}${C.reset}\n`);
|
|
491
|
+
logEmergency('ELEVATED', `Alert issued: ${message}`, { alertId: result.alert_id });
|
|
492
|
+
} else {
|
|
493
|
+
console.log(` ${C.yellow}⚠ Alert submitted (check response):${C.reset}`);
|
|
494
|
+
console.log(` ${JSON.stringify(result)}\n`);
|
|
495
|
+
}
|
|
496
|
+
} catch (err) {
|
|
497
|
+
console.log(` ${C.yellow}⚠ Could not reach governance endpoint (network may be down)${C.reset}`);
|
|
498
|
+
console.log(` ${C.dim}Storing alert locally for later submission...${C.reset}\n`);
|
|
499
|
+
logEmergency('ELEVATED', `Local alert (network unreachable): ${message}`, { error: err.message });
|
|
500
|
+
}
|
|
501
|
+
}
|
|
502
|
+
|
|
503
|
+
async function emergencyFailover(opts) {
|
|
504
|
+
const { rpc, json } = opts;
|
|
505
|
+
console.log(`\n${C.bright}${C.magenta}⚡ Aether Emergency Failover${C.reset}\n`);
|
|
506
|
+
|
|
507
|
+
const identity = getValidatorConfig();
|
|
508
|
+
if (!identity) {
|
|
509
|
+
console.log(` ${C.red}✗ No validator identity found.${C.reset}`);
|
|
510
|
+
console.log(` ${C.dim} Run:${C.reset} ${C.cyan}aether init${C.reset} first to configure validator\n`);
|
|
511
|
+
return;
|
|
512
|
+
}
|
|
513
|
+
|
|
514
|
+
console.log(` ${C.dim}Validator:${C.reset} ${identity.identity ?? identity.nodeId ?? 'unknown'}`);
|
|
515
|
+
console.log(` ${C.dim}Checking backup node status...${C.reset}\n`);
|
|
516
|
+
|
|
517
|
+
// Check if backup RPC is configured
|
|
518
|
+
const backupRpc = process.env.AETHER_BACKUP_RPC;
|
|
519
|
+
if (!backupRpc) {
|
|
520
|
+
console.log(` ${C.yellow}⚠ AETHER_BACKUP_RPC not set.${C.reset}`);
|
|
521
|
+
console.log(` ${C.dim} Export it in your environment to enable automatic failover.${C.reset}`);
|
|
522
|
+
console.log(` ${C.dim} export AETHER_BACKUP_RPC=http://backup-node:8899${C.reset}\n`);
|
|
523
|
+
}
|
|
524
|
+
|
|
525
|
+
// Check current node
|
|
526
|
+
const [health, slotHalt] = await Promise.all([
|
|
527
|
+
checkNodeHealth(rpc),
|
|
528
|
+
checkSlotProgression(rpc, 3),
|
|
529
|
+
]);
|
|
530
|
+
|
|
531
|
+
console.log(` ${C.dim}Current node health:${C.reset} ${health.ok ? `${C.green}✓${C.reset}` : `${C.red}✗${C.reset}`}`);
|
|
532
|
+
console.log(` ${C.dim}Slot status:${C.reset} ${slotHalt.halted ? `${C.red}HALTED${C.reset}` : `${C.green}Advancing${C.reset}`}\n`);
|
|
533
|
+
|
|
534
|
+
if (!health.ok || slotHalt.halted) {
|
|
535
|
+
console.log(` ${C.red}⚠ Primary node is down! Attempting failover...${C.reset}\n`);
|
|
536
|
+
|
|
537
|
+
if (backupRpc) {
|
|
538
|
+
try {
|
|
539
|
+
const backupHealth = await checkNodeHealth(backupRpc);
|
|
540
|
+
if (backupHealth.ok) {
|
|
541
|
+
console.log(` ${C.green}✓ Backup node is healthy!${C.reset}`);
|
|
542
|
+
console.log(` ${C.green}✓ Failover would succeed.${C.reset}`);
|
|
543
|
+
console.log(` ${C.dim} Update your AETHER_RPC to:${C.reset} ${backupRpc}\n`);
|
|
544
|
+
logEmergency('CRITICAL', 'Failover needed and backup available', { backupRpc });
|
|
545
|
+
} else {
|
|
546
|
+
console.log(` ${C.red}✗ Backup node is also unreachable.${C.reset}\n`);
|
|
547
|
+
logEmergency('CRITICAL', 'Failover failed - both primary and backup down', {});
|
|
548
|
+
}
|
|
549
|
+
} catch {
|
|
550
|
+
console.log(` ${C.red}✗ Backup node check failed.${C.reset}\n`);
|
|
551
|
+
}
|
|
552
|
+
} else {
|
|
553
|
+
console.log(` ${C.yellow}⚠ Set AETHER_BACKUP_RPC to enable automatic failover.${C.reset}\n`);
|
|
554
|
+
logEmergency('CRITICAL', 'Failover needed but no backup configured', {});
|
|
555
|
+
}
|
|
556
|
+
} else {
|
|
557
|
+
console.log(` ${C.green}✓ Primary node is healthy. No failover needed.${C.reset}\n`);
|
|
558
|
+
}
|
|
559
|
+
}
|
|
560
|
+
|
|
561
|
+
async function emergencyHistory(opts) {
|
|
562
|
+
const { json, lines = 20 } = opts;
|
|
563
|
+
const events = readEmergencyLog(lines);
|
|
564
|
+
|
|
565
|
+
if (events.length === 0) {
|
|
566
|
+
console.log(`\n${C.dim}No emergency events logged.${C.reset}\n`);
|
|
567
|
+
return;
|
|
568
|
+
}
|
|
569
|
+
|
|
570
|
+
if (json) {
|
|
571
|
+
console.log(JSON.stringify(events, null, 2));
|
|
572
|
+
return;
|
|
573
|
+
}
|
|
574
|
+
|
|
575
|
+
console.log(`\n${C.bright}📋 Recent Emergency Events${C.reset} ${C.dim}(last ${events.length})${C.reset}\n`);
|
|
576
|
+
|
|
577
|
+
for (const ev of events) {
|
|
578
|
+
const levelColor = ev.level === 'OK' ? C.green : ev.level === 'WARNING' ? C.yellow : ev.level === 'ELEVATED' ? C.magenta : C.red;
|
|
579
|
+
const ts = ev.timestamp ? ev.timestamp.substring(0, 19) : '?';
|
|
580
|
+
console.log(` ${levelColor}[${ev.level}]${C.reset} ${C.dim}${ts}${C.reset} — ${ev.message}`);
|
|
581
|
+
}
|
|
582
|
+
console.log();
|
|
583
|
+
}
|
|
584
|
+
|
|
585
|
+
// ---------------------------------------------------------------------------
|
|
586
|
+
// CLI argument parsing
|
|
587
|
+
// ---------------------------------------------------------------------------
|
|
588
|
+
|
|
589
|
+
function parseArgs() {
|
|
590
|
+
const args = process.argv.slice(3); // [node, index.js, emergency, <subcmd>, ...]
|
|
591
|
+
return args;
|
|
592
|
+
}
|
|
593
|
+
|
|
594
|
+
async function main() {
|
|
595
|
+
const rawArgs = parseArgs();
|
|
596
|
+
const subcmd = rawArgs[0] || 'status';
|
|
597
|
+
|
|
598
|
+
const allArgs = rawArgs.slice(1);
|
|
599
|
+
const rpcIndex = allArgs.findIndex(a => a === '--rpc');
|
|
600
|
+
const rpc = rpcIndex !== -1 && allArgs[rpcIndex + 1] ? allArgs[rpcIndex + 1] : DEFAULT_RPC;
|
|
601
|
+
|
|
602
|
+
const opts = {
|
|
603
|
+
rpc,
|
|
604
|
+
json: allArgs.includes('--json'),
|
|
605
|
+
message: null,
|
|
606
|
+
interval: 30,
|
|
607
|
+
lines: 20,
|
|
608
|
+
};
|
|
609
|
+
|
|
610
|
+
const msgIndex = allArgs.findIndex(a => a === '--message');
|
|
611
|
+
if (msgIndex !== -1 && allArgs[msgIndex + 1]) opts.message = allArgs[msgIndex + 1];
|
|
612
|
+
|
|
613
|
+
const intIndex = allArgs.findIndex(a => a === '--interval');
|
|
614
|
+
if (intIndex !== -1 && allArgs[intIndex + 1]) opts.interval = parseInt(allArgs[intIndex + 1], 10);
|
|
615
|
+
|
|
616
|
+
const linesIndex = allArgs.findIndex(a => a === '--lines');
|
|
617
|
+
if (linesIndex !== -1 && allArgs[linesIndex + 1]) opts.lines = parseInt(allArgs[linesIndex + 1], 10);
|
|
618
|
+
|
|
619
|
+
switch (subcmd) {
|
|
620
|
+
case 'status':
|
|
621
|
+
await emergencyStatus(opts);
|
|
622
|
+
break;
|
|
623
|
+
case 'monitor':
|
|
624
|
+
await emergencyMonitor(opts);
|
|
625
|
+
break;
|
|
626
|
+
case 'check':
|
|
627
|
+
await emergencyCheck(opts);
|
|
628
|
+
break;
|
|
629
|
+
case 'alert':
|
|
630
|
+
await emergencyAlert(opts);
|
|
631
|
+
break;
|
|
632
|
+
case 'failover':
|
|
633
|
+
await emergencyFailover(opts);
|
|
634
|
+
break;
|
|
635
|
+
case 'history':
|
|
636
|
+
await emergencyHistory(opts);
|
|
637
|
+
break;
|
|
638
|
+
default:
|
|
639
|
+
console.log(`\n${C.bright}${C.cyan}aether emergency${C.reset} — Emergency Response & Alert System\n`);
|
|
640
|
+
console.log(`Usage: ${C.cyan}aether emergency <command>${C.reset}\n`);
|
|
641
|
+
console.log(`Commands:`);
|
|
642
|
+
console.log(` ${C.cyan}status${C.reset} Check current emergency level (default)`);
|
|
643
|
+
console.log(` ${C.cyan}monitor${C.reset} Continuous monitoring loop (--interval <sec>)`);
|
|
644
|
+
console.log(` ${C.cyan}check${C.reset} Run full diagnostic checks`);
|
|
645
|
+
console.log(` ${C.cyan}alert${C.reset} Issue a governance alert (--message "...")`);
|
|
646
|
+
console.log(` ${C.cyan}failover${C.reset} Trigger backup node failover`);
|
|
647
|
+
console.log(` ${C.cyan}history${C.reset} Show recent emergency events (--lines <n>)`);
|
|
648
|
+
console.log(`\nOptions:`);
|
|
649
|
+
console.log(` --rpc <url> RPC endpoint (default: $AETHER_RPC or localhost)`);
|
|
650
|
+
console.log(` --json JSON output\n`);
|
|
651
|
+
}
|
|
652
|
+
}
|
|
653
|
+
|
|
654
|
+
main().catch(err => {
|
|
655
|
+
console.error(`\n${C.red}Error in emergency command:${C.reset}`, err.message, '\n');
|
|
656
|
+
process.exit(1);
|
|
657
|
+
});
|