@orb44/cli 0.1.6 → 0.1.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +4 -4
- package/bin/orb44.mjs +3 -2
- package/package.json +1 -1
- package/server/pulse.mjs +339 -12
package/README.md
CHANGED
|
@@ -13,11 +13,11 @@ npx @orb44/cli login --url https://orb44.com
|
|
|
13
13
|
Open the printed link in a browser where you are already signed in, confirm the domain, then optionally install the systemd daemon (default no). After a global install the command is `orb44`.
|
|
14
14
|
|
|
15
15
|
```bash
|
|
16
|
-
npx @orb44/cli@0.1.
|
|
17
|
-
npx @orb44/cli@0.1.
|
|
18
|
-
npx @orb44/cli@0.1.
|
|
16
|
+
npx @orb44/cli@0.1.7 status
|
|
17
|
+
npx @orb44/cli@0.1.7 pulse
|
|
18
|
+
npx @orb44/cli@0.1.7 install # later: background pulse, starts after reboot
|
|
19
19
|
orb44 update # after install: replace /usr/lib/orb44-sat from npm
|
|
20
|
-
npx @orb44/cli@0.1.
|
|
20
|
+
npx @orb44/cli@0.1.7 logout
|
|
21
21
|
```
|
|
22
22
|
|
|
23
23
|
Do not run bare `npx @orb44/cli` — it can reuse an old cache. Pin the version or use `orb44` after install.
|
package/bin/orb44.mjs
CHANGED
|
@@ -122,7 +122,7 @@ async function ensureLang(opts) {
|
|
|
122
122
|
return lang;
|
|
123
123
|
}
|
|
124
124
|
lang = detectLang();
|
|
125
|
-
if (opts.yes || !input.isTTY || !output.isTTY) {
|
|
125
|
+
if (opts.yes || opts.url || !input.isTTY || !output.isTTY) {
|
|
126
126
|
saveCli({ ...loadCli(), lang });
|
|
127
127
|
return lang;
|
|
128
128
|
}
|
|
@@ -242,9 +242,10 @@ async function revokeLocalDevices(devices, fallbackApi) {
|
|
|
242
242
|
}
|
|
243
243
|
|
|
244
244
|
async function cmdLogin(opts) {
|
|
245
|
+
const url = String(opts.url || process.env.ORB44_API || "http://127.0.0.1:8787").replace(/\/$/, "");
|
|
246
|
+
process.stdout.write(`orb44 login → ${url}\n`);
|
|
245
247
|
await ensureLang(opts);
|
|
246
248
|
banner();
|
|
247
|
-
const url = String(opts.url || process.env.ORB44_API || "http://127.0.0.1:8787").replace(/\/$/, "");
|
|
248
249
|
const found = existingOnBox();
|
|
249
250
|
if (hasExistingInstall(found)) {
|
|
250
251
|
const replace = await wantReplace(found, opts);
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@orb44/cli",
|
|
3
|
-
"version": "0.1.
|
|
3
|
+
"version": "0.1.7",
|
|
4
4
|
"description": "Orb44: street snapshot of your site. This CLI is the Watch satellite — pulse from the host (load, listeners, hardening). The cabinet does not run commands on the machine.",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"bin": {
|
package/server/pulse.mjs
CHANGED
|
@@ -78,17 +78,26 @@ function run(cmd, args) {
|
|
|
78
78
|
|
|
79
79
|
const SKIP_TOP = new Set(["ps", "ss", "lsof"]);
|
|
80
80
|
|
|
81
|
+
const STAT_RE = /^[DRSTZIX][A-Za-z+<]*$/;
|
|
82
|
+
|
|
81
83
|
export function parsePsTop(text) {
|
|
82
84
|
const rows = [];
|
|
83
85
|
for (const line of String(text || "").split("\n")) {
|
|
84
|
-
const m = line.trim().match(/^(\d+)\s+([\d.]+)\s+(\d+)\s+(\S+)
|
|
86
|
+
const m = line.trim().match(/^(\d+)\s+([\d.]+)\s+(\d+)\s+(\S+)(?:\s+(\S+))?$/);
|
|
85
87
|
if (!m) continue;
|
|
86
|
-
|
|
88
|
+
let stat = null;
|
|
89
|
+
let commRaw = m[4];
|
|
90
|
+
if (m[5] && STAT_RE.test(m[4]) && m[4].length <= 8) {
|
|
91
|
+
stat = m[4].slice(0, 4);
|
|
92
|
+
commRaw = m[5];
|
|
93
|
+
}
|
|
94
|
+
const comm = sanitizeComm(commRaw);
|
|
87
95
|
if (!comm || SKIP_TOP.has(comm)) continue;
|
|
88
96
|
rows.push({
|
|
89
97
|
comm,
|
|
90
98
|
cpuPct: Math.round(Number(m[2]) * 10) / 10,
|
|
91
99
|
rssMb: Math.round(Number(m[3]) / 1024),
|
|
100
|
+
stat,
|
|
92
101
|
});
|
|
93
102
|
}
|
|
94
103
|
rows.sort((a, b) => b.cpuPct - a.cpuPct || b.rssMb - a.rssMb);
|
|
@@ -168,8 +177,10 @@ function listenTable() {
|
|
|
168
177
|
}
|
|
169
178
|
|
|
170
179
|
function topTable() {
|
|
171
|
-
const
|
|
172
|
-
|
|
180
|
+
const withStat = run("ps", ["-axo", "pid=,pcpu=,rss=,stat=,comm="]);
|
|
181
|
+
const parsed = parsePsTop(withStat);
|
|
182
|
+
if (parsed.length) return parsed;
|
|
183
|
+
return parsePsTop(run("ps", ["-axo", "pid=,pcpu=,rss=,comm="]));
|
|
173
184
|
}
|
|
174
185
|
|
|
175
186
|
function okListenAddr(a) {
|
|
@@ -214,6 +225,7 @@ export function sanitizePulse(raw = {}) {
|
|
|
214
225
|
comm: sanitizeComm(r.comm),
|
|
215
226
|
cpuPct: Math.max(0, Math.min(100, Number(r.cpuPct) || 0)),
|
|
216
227
|
rssMb: Math.max(0, Math.round(Number(r.rssMb) || 0)),
|
|
228
|
+
stat: STAT_RE.test(String(r.stat || "")) ? String(r.stat).slice(0, 4) : null,
|
|
217
229
|
}))
|
|
218
230
|
.filter((r) => r.comm)
|
|
219
231
|
: [];
|
|
@@ -253,6 +265,7 @@ export function sanitizePulse(raw = {}) {
|
|
|
253
265
|
};
|
|
254
266
|
const limited = Boolean(raw.limited);
|
|
255
267
|
const oom = Boolean(raw.oom) || hardening.oomKills > 0;
|
|
268
|
+
const pressure = sanitizePressure(raw.pressure || raw);
|
|
256
269
|
const base = {
|
|
257
270
|
ts: Number(raw.ts) || Date.now(),
|
|
258
271
|
hostname: sanitizeComm(raw.hostname) || os.hostname().slice(0, 40),
|
|
@@ -267,6 +280,7 @@ export function sanitizePulse(raw = {}) {
|
|
|
267
280
|
oom,
|
|
268
281
|
limited,
|
|
269
282
|
hardening,
|
|
283
|
+
pressure,
|
|
270
284
|
};
|
|
271
285
|
return { ...base, gradeInside: gradeInside(base) };
|
|
272
286
|
}
|
|
@@ -351,6 +365,172 @@ export function parseVmstatOom(text) {
|
|
|
351
365
|
return m ? Number(m[1]) : 0;
|
|
352
366
|
}
|
|
353
367
|
|
|
368
|
+
export function parseSockstat(text) {
|
|
369
|
+
const tcp = /(?:^|\n)TCP:\s+inuse\s+(\d+)(?:\s+orphan\s+(\d+))?(?:\s+tw\s+(\d+))?/i.exec(String(text || ""));
|
|
370
|
+
if (!tcp) return { tcpInuse: null, tcpOrphan: null, tcpTw: null };
|
|
371
|
+
return {
|
|
372
|
+
tcpInuse: Number(tcp[1]),
|
|
373
|
+
tcpOrphan: tcp[2] != null ? Number(tcp[2]) : null,
|
|
374
|
+
tcpTw: tcp[3] != null ? Number(tcp[3]) : null,
|
|
375
|
+
};
|
|
376
|
+
}
|
|
377
|
+
|
|
378
|
+
export function parseFileNr(text) {
|
|
379
|
+
const p = String(text || "").trim().split(/[\s,]+/);
|
|
380
|
+
const used = Number(p[0]);
|
|
381
|
+
const max = Number(p[2] ?? p[1]);
|
|
382
|
+
if (!Number.isFinite(used) || !Number.isFinite(max) || max <= 0) return { fileUsed: null, fileMax: null, filePct: null };
|
|
383
|
+
return { fileUsed: used, fileMax: max, filePct: Math.max(0, Math.min(100, Math.round((used / max) * 100))) };
|
|
384
|
+
}
|
|
385
|
+
|
|
386
|
+
export function parseChronyTracking(text) {
|
|
387
|
+
const m =
|
|
388
|
+
/Last offset\s*:\s*([+-]?\d+(?:\.\d+)?)\s*seconds/i.exec(String(text || "")) ||
|
|
389
|
+
/System time\s*:\s*(\d+(?:\.\d+)?)\s*seconds\s+(slow|fast)/i.exec(String(text || ""));
|
|
390
|
+
if (!m) return null;
|
|
391
|
+
let n = Number(m[1]);
|
|
392
|
+
if (!Number.isFinite(n)) return null;
|
|
393
|
+
if (m[2] === "fast") n = Math.abs(n);
|
|
394
|
+
if (m[2] === "slow") n = -Math.abs(n);
|
|
395
|
+
return Math.round(n * 1000) / 1000;
|
|
396
|
+
}
|
|
397
|
+
|
|
398
|
+
export function parseCgroupEvents(text) {
|
|
399
|
+
const kill = /(?:^|\n)oom_kill\s+(\d+)/.exec(String(text || ""));
|
|
400
|
+
const oom = /(?:^|\n)oom\s+(\d+)/.exec(String(text || ""));
|
|
401
|
+
return {
|
|
402
|
+
cgroupOom: kill ? Number(kill[1]) : oom ? Number(oom[1]) : 0,
|
|
403
|
+
memFailcnt: 0,
|
|
404
|
+
};
|
|
405
|
+
}
|
|
406
|
+
|
|
407
|
+
export function parseNetstatListenOverflows(text) {
|
|
408
|
+
const lines = String(text || "").split("\n");
|
|
409
|
+
let header = "";
|
|
410
|
+
let values = "";
|
|
411
|
+
for (const line of lines) {
|
|
412
|
+
if (/^TcpExt:\s+/.test(line) && /ListenOverflows/.test(line)) header = line;
|
|
413
|
+
else if (/^TcpExt:\s+/.test(line) && header && !/ListenOverflows/.test(line)) values = line;
|
|
414
|
+
}
|
|
415
|
+
if (!header || !values) return { listenOverflows: null, listenDrops: null };
|
|
416
|
+
const keys = header.replace(/^TcpExt:\s+/, "").trim().split(/\s+/);
|
|
417
|
+
const nums = values.replace(/^TcpExt:\s+/, "").trim().split(/\s+/).map(Number);
|
|
418
|
+
const idx = (name) => keys.indexOf(name);
|
|
419
|
+
const n = (i) => (i >= 0 && Number.isFinite(nums[i]) ? nums[i] : null);
|
|
420
|
+
return { listenOverflows: n(idx("ListenOverflows")), listenDrops: n(idx("ListenDrops")) };
|
|
421
|
+
}
|
|
422
|
+
|
|
423
|
+
function readTrim(p) {
|
|
424
|
+
try {
|
|
425
|
+
return fs.readFileSync(p, "utf8").trim();
|
|
426
|
+
} catch {
|
|
427
|
+
return "";
|
|
428
|
+
}
|
|
429
|
+
}
|
|
430
|
+
|
|
431
|
+
function numFile(p) {
|
|
432
|
+
const n = Number(readTrim(p));
|
|
433
|
+
return Number.isFinite(n) ? n : null;
|
|
434
|
+
}
|
|
435
|
+
|
|
436
|
+
export function stuckFromTop(top = []) {
|
|
437
|
+
return (top || []).filter((r) => {
|
|
438
|
+
const s = String(r.stat || "")[0];
|
|
439
|
+
return s === "D" || s === "Z" || s === "T";
|
|
440
|
+
}).slice(0, 6);
|
|
441
|
+
}
|
|
442
|
+
|
|
443
|
+
export function hotFromTop(top = []) {
|
|
444
|
+
return (top || []).filter((r) => Number(r.cpuPct) >= 70 || Number(r.rssMb) >= 1024).slice(0, 6);
|
|
445
|
+
}
|
|
446
|
+
|
|
447
|
+
export function cpuCount(pulse) {
|
|
448
|
+
const n = Number(pulse?.pressure?.cpus);
|
|
449
|
+
return Number.isFinite(n) && n > 0 ? n : 2;
|
|
450
|
+
}
|
|
451
|
+
|
|
452
|
+
function collectPressure() {
|
|
453
|
+
const cpus = Math.max(1, os.cpus()?.length || 1);
|
|
454
|
+
const sock = parseSockstat(readTrim("/proc/net/sockstat"));
|
|
455
|
+
const files = parseFileNr(readTrim("/proc/sys/fs/file-nr"));
|
|
456
|
+
let conntrackUsed = numFile("/proc/sys/net/netfilter/nf_conntrack_count");
|
|
457
|
+
let conntrackMax = numFile("/proc/sys/net/netfilter/nf_conntrack_max");
|
|
458
|
+
if (conntrackUsed == null) conntrackUsed = numFile("/proc/sys/net/nf_conntrack_count");
|
|
459
|
+
if (conntrackMax == null) conntrackMax = numFile("/proc/sys/net/nf_conntrack_max");
|
|
460
|
+
const conntrackPct =
|
|
461
|
+
conntrackUsed != null && conntrackMax > 0 ? Math.max(0, Math.min(100, Math.round((conntrackUsed / conntrackMax) * 100))) : null;
|
|
462
|
+
const overflows = parseNetstatListenOverflows(readTrim("/proc/net/netstat"));
|
|
463
|
+
let clockOffsetSec = parseChronyTracking(runOut("chronyc", ["tracking"]));
|
|
464
|
+
let ntpSync = null;
|
|
465
|
+
if (clockOffsetSec != null) ntpSync = true;
|
|
466
|
+
else {
|
|
467
|
+
const td = runOut("timedatectl", ["show", "-p", "NTPSynchronized", "--value"]);
|
|
468
|
+
if (/^yes$/i.test(td.trim())) ntpSync = true;
|
|
469
|
+
else if (/^no$/i.test(td.trim())) ntpSync = false;
|
|
470
|
+
}
|
|
471
|
+
let cgroupOom = 0;
|
|
472
|
+
let memFailcnt = 0;
|
|
473
|
+
const ev = parseCgroupEvents(readTrim("/sys/fs/cgroup/memory.events") || readTrim("/sys/fs/cgroup/memory/memory.events"));
|
|
474
|
+
cgroupOom = ev.cgroupOom || 0;
|
|
475
|
+
memFailcnt = numFile("/sys/fs/cgroup/memory/memory.failcnt") ?? ev.memFailcnt ?? 0;
|
|
476
|
+
return {
|
|
477
|
+
cpus,
|
|
478
|
+
conntrackUsed,
|
|
479
|
+
conntrackMax,
|
|
480
|
+
conntrackPct,
|
|
481
|
+
tcpInuse: sock.tcpInuse,
|
|
482
|
+
tcpTw: sock.tcpTw,
|
|
483
|
+
tcpOrphan: sock.tcpOrphan,
|
|
484
|
+
listenOverflows: overflows.listenOverflows,
|
|
485
|
+
listenDrops: overflows.listenDrops,
|
|
486
|
+
fileUsed: files.fileUsed,
|
|
487
|
+
fileMax: files.fileMax,
|
|
488
|
+
filePct: files.filePct,
|
|
489
|
+
clockOffsetSec,
|
|
490
|
+
ntpSync,
|
|
491
|
+
memFailcnt: Math.max(0, Math.min(999999, Number(memFailcnt) || 0)),
|
|
492
|
+
cgroupOom: Math.max(0, Math.min(999, Number(cgroupOom) || 0)),
|
|
493
|
+
};
|
|
494
|
+
}
|
|
495
|
+
|
|
496
|
+
function sanitizePressure(raw = {}) {
|
|
497
|
+
const n = (v, lo, hi) => {
|
|
498
|
+
if (v == null || v === "") return null;
|
|
499
|
+
const x = Number(v);
|
|
500
|
+
if (!Number.isFinite(x)) return null;
|
|
501
|
+
return Math.max(lo, Math.min(hi, x));
|
|
502
|
+
};
|
|
503
|
+
const cpus = n(raw.cpus, 1, 512) || null;
|
|
504
|
+
const conntrackUsed = n(raw.conntrackUsed, 0, 1e9);
|
|
505
|
+
const conntrackMax = n(raw.conntrackMax, 0, 1e9);
|
|
506
|
+
const conntrackPct =
|
|
507
|
+
raw.conntrackPct != null
|
|
508
|
+
? n(raw.conntrackPct, 0, 100)
|
|
509
|
+
: conntrackUsed != null && conntrackMax > 0
|
|
510
|
+
? Math.max(0, Math.min(100, Math.round((conntrackUsed / conntrackMax) * 100)))
|
|
511
|
+
: null;
|
|
512
|
+
const clockOffsetSec = n(raw.clockOffsetSec, -86400, 86400);
|
|
513
|
+
const ntpSync = raw.ntpSync === true || raw.ntpSync === false ? raw.ntpSync : null;
|
|
514
|
+
return {
|
|
515
|
+
cpus,
|
|
516
|
+
conntrackUsed,
|
|
517
|
+
conntrackMax,
|
|
518
|
+
conntrackPct,
|
|
519
|
+
tcpInuse: n(raw.tcpInuse, 0, 1e9),
|
|
520
|
+
tcpTw: n(raw.tcpTw, 0, 1e9),
|
|
521
|
+
tcpOrphan: n(raw.tcpOrphan, 0, 1e9),
|
|
522
|
+
listenOverflows: n(raw.listenOverflows, 0, 1e12),
|
|
523
|
+
listenDrops: n(raw.listenDrops, 0, 1e12),
|
|
524
|
+
fileUsed: n(raw.fileUsed, 0, 1e12),
|
|
525
|
+
fileMax: n(raw.fileMax, 0, 1e12),
|
|
526
|
+
filePct: n(raw.filePct, 0, 100),
|
|
527
|
+
clockOffsetSec,
|
|
528
|
+
ntpSync,
|
|
529
|
+
memFailcnt: n(raw.memFailcnt, 0, 1e9) || 0,
|
|
530
|
+
cgroupOom: n(raw.cgroupOom, 0, 999) || 0,
|
|
531
|
+
};
|
|
532
|
+
}
|
|
533
|
+
|
|
354
534
|
export function insideServiceWorldPorts(pulse) {
|
|
355
535
|
return [...new Set((pulse?.listen || []).filter((r) => exposed(r.addr) && ADMIN_PORTS.has(r.port)).map((r) => r.port))];
|
|
356
536
|
}
|
|
@@ -425,19 +605,23 @@ export function parseFail2banJail(text) {
|
|
|
425
605
|
export function gradeInside(pulse = {}) {
|
|
426
606
|
const listen = pulse.listen || [];
|
|
427
607
|
const h = pulse.hardening || {};
|
|
608
|
+
const pr = pulse.pressure || {};
|
|
428
609
|
const exposedAdmin = listen.filter((r) => exposed(r.addr) && ADMIN_PORTS.has(r.port));
|
|
429
610
|
const fw = Boolean(h.firewall);
|
|
430
611
|
const ban = Boolean(h.fail2ban);
|
|
431
612
|
const updates = Boolean(h.updates);
|
|
432
613
|
const diskHigh = Number(pulse.diskUsedPct) >= 85;
|
|
433
|
-
const oom = Boolean(pulse.oom) || Number(h.oomKills) > 0;
|
|
614
|
+
const oom = Boolean(pulse.oom) || Number(h.oomKills) > 0 || Number(pr.cgroupOom) > 0;
|
|
615
|
+
const loadHigh = Number(pulse.load1) >= Math.max(2, cpuCount(pulse));
|
|
616
|
+
const connHot = Number(pr.conntrackPct) >= 90;
|
|
617
|
+
const stuck = stuckFromTop(pulse.top).length > 0;
|
|
434
618
|
if (h.dockerApi) return "D";
|
|
435
619
|
if (h.sshPassword && h.sshRoot && h.sshWorld) return "D";
|
|
436
620
|
if (!fw && exposedAdmin.length) return "D";
|
|
437
621
|
if (!fw && !ban) return "D";
|
|
438
622
|
if (exposedAdmin.length || h.sshPassword || h.sshRoot) return "C";
|
|
439
623
|
if (!fw || !ban) return "C";
|
|
440
|
-
if (diskHigh || oom || h.rebootNeeded || !h.timesync) return "C";
|
|
624
|
+
if (diskHigh || oom || h.rebootNeeded || !h.timesync || loadHigh || connHot || stuck) return "C";
|
|
441
625
|
if (fw && ban && updates && h.timesync && !h.sshPassword && !h.sshRoot && !exposedAdmin.length && !h.dockerApi) return "A";
|
|
442
626
|
if (fw && ban && !exposedAdmin.length && !h.dockerApi) return "B";
|
|
443
627
|
return "C";
|
|
@@ -448,7 +632,7 @@ function labelPort(r) {
|
|
|
448
632
|
return svc ? `${svc} :${r.port}` : `:${r.port}`;
|
|
449
633
|
}
|
|
450
634
|
|
|
451
|
-
export function compareInside(pulse, rec = {}) {
|
|
635
|
+
export function compareInside(pulse, rec = {}, prev = null) {
|
|
452
636
|
const notes = [];
|
|
453
637
|
const listen = pulse?.listen || [];
|
|
454
638
|
const streetIp = rec.ticket?.ip || rec.watch?.current?.ip || null;
|
|
@@ -488,8 +672,9 @@ export function compareInside(pulse, rec = {}) {
|
|
|
488
672
|
do: "Сверьте DNS A с тем VPS, куда поставили сателлит. Если сайт за Cloudflare — это ожидаемо, не инцидент.",
|
|
489
673
|
});
|
|
490
674
|
}
|
|
491
|
-
const loadHigh = Number(pulse?.load1) >= Math.max(2, (
|
|
492
|
-
const memHigh = pulse?.memTotal && pulse.memUsed / pulse.memTotal >= 0.
|
|
675
|
+
const loadHigh = Number(pulse?.load1) >= Math.max(2, cpuCount(pulse));
|
|
676
|
+
const memHigh = pulse?.memTotal && pulse.memUsed / pulse.memTotal >= 0.85;
|
|
677
|
+
const pr = pulse?.pressure || {};
|
|
493
678
|
if (loadHigh || memHigh) {
|
|
494
679
|
const top = pulse.top?.[0];
|
|
495
680
|
const who = top ? `${top.comm} ${top.cpuPct}%` : "процесс в топе не виден";
|
|
@@ -605,6 +790,93 @@ export function compareInside(pulse, rec = {}) {
|
|
|
605
790
|
});
|
|
606
791
|
}
|
|
607
792
|
}
|
|
793
|
+
const stuck = stuckFromTop(pulse?.top);
|
|
794
|
+
if (stuck.length) {
|
|
795
|
+
const who = stuck.map((r) => `${r.comm} (${r.stat || "?"})`).join(", ");
|
|
796
|
+
notes.push({
|
|
797
|
+
kind: "stuck-proc",
|
|
798
|
+
title: "Процессы зависли или зомби",
|
|
799
|
+
text: `Состояние D/Z/T: ${who}. Так машина стоит на диске или мёртвых воркерах, а не «просто высокая нагрузка».`,
|
|
800
|
+
do: "Не убивайте с кабинета — его нет. На сервере: ps и диск, не WAF. Orb44 процессы сам не трогает.",
|
|
801
|
+
});
|
|
802
|
+
}
|
|
803
|
+
const hot = hotFromTop(pulse?.top);
|
|
804
|
+
if (hot.length && !loadHigh) {
|
|
805
|
+
const who = hot.map((r) => `${r.comm} ${r.cpuPct}%/${r.rssMb}M`).join(", ");
|
|
806
|
+
notes.push({
|
|
807
|
+
kind: "hot-proc",
|
|
808
|
+
title: "Сервис жрёт CPU или RAM",
|
|
809
|
+
text: `Топ без общей перегрузки load: ${who}. Воркер уже упёрся, витрина может ещё отвечать.`,
|
|
810
|
+
do: "Смотрите этот процесс (php-fpm, node, mysql). Orb44 его не рестартит.",
|
|
811
|
+
});
|
|
812
|
+
}
|
|
813
|
+
if (Number(pr.conntrackPct) >= 80) {
|
|
814
|
+
notes.push({
|
|
815
|
+
kind: "conntrack",
|
|
816
|
+
title: "Таблица соединений почти полная",
|
|
817
|
+
text: `conntrack ${pr.conntrackUsed}/${pr.conntrackMax} (${pr.conntrackPct}%). Новые сессии начнут отбрасываться — с улицы это 502, изнутри это очередь.`,
|
|
818
|
+
do: "Ищите кто держит кучу TCP. Не открывайте порты «чтобы помогло».",
|
|
819
|
+
});
|
|
820
|
+
}
|
|
821
|
+
const prevPr = prev?.pressure || {};
|
|
822
|
+
const overDelta =
|
|
823
|
+
pr.listenOverflows != null && prevPr.listenOverflows != null
|
|
824
|
+
? Number(pr.listenOverflows) - Number(prevPr.listenOverflows)
|
|
825
|
+
: 0;
|
|
826
|
+
const dropDelta =
|
|
827
|
+
pr.listenDrops != null && prevPr.listenDrops != null
|
|
828
|
+
? Number(pr.listenDrops) - Number(prevPr.listenDrops)
|
|
829
|
+
: 0;
|
|
830
|
+
if (overDelta > 0 || dropDelta > 0) {
|
|
831
|
+
notes.push({
|
|
832
|
+
kind: "backlog",
|
|
833
|
+
title: "Очередь accept переполняется",
|
|
834
|
+
text: `С прошлого пульса ListenOverflows +${overDelta}, ListenDrops +${dropDelta}. Сервис не успевает брать соединения.`,
|
|
835
|
+
do: "Больше воркеров или меньше входа. Orb44 лимиты сам не поднимает.",
|
|
836
|
+
});
|
|
837
|
+
}
|
|
838
|
+
if (Number(pr.filePct) >= 85) {
|
|
839
|
+
notes.push({
|
|
840
|
+
kind: "files",
|
|
841
|
+
title: "Заканчиваются файловые дескрипторы",
|
|
842
|
+
text: `Открыто ${pr.filePct}% лимита (${pr.fileUsed}/${pr.fileMax}). Типичный «внезапно не открывается сокет».`,
|
|
843
|
+
do: "Кто держит файлы: воркер или утечка. Orb44 ulimit сам не меняет.",
|
|
844
|
+
});
|
|
845
|
+
}
|
|
846
|
+
if (pr.clockOffsetSec != null && Math.abs(pr.clockOffsetSec) >= 5) {
|
|
847
|
+
notes.push({
|
|
848
|
+
kind: "clock",
|
|
849
|
+
title: "Часы машины уехали",
|
|
850
|
+
text: `Смещение NTP ${pr.clockOffsetSec} с. TLS и метки Watch начнут врать.`,
|
|
851
|
+
do: "Почините chrony/timesyncd. Orb44 время сам не ставит.",
|
|
852
|
+
});
|
|
853
|
+
}
|
|
854
|
+
if (Number(pr.cgroupOom) > 0 || Number(pr.memFailcnt) > 20) {
|
|
855
|
+
notes.push({
|
|
856
|
+
kind: "cgroup-oom",
|
|
857
|
+
title: "Контейнер или cgroup упирается в память",
|
|
858
|
+
text: `cgroup oom=${pr.cgroupOom || 0}, failcnt=${pr.memFailcnt || 0}. Это не journal — счётчик ядра.`,
|
|
859
|
+
do: "Лимит памяти контейнера, не WAF. Orb44 лимиты сам не поднимает.",
|
|
860
|
+
});
|
|
861
|
+
}
|
|
862
|
+
if (prev?.listen && pulse?.listen) {
|
|
863
|
+
const face = (p) =>
|
|
864
|
+
(p.listen || [])
|
|
865
|
+
.filter((r) => Number(r.port) === 80 || Number(r.port) === 443)
|
|
866
|
+
.map((r) => `${r.port}:${r.comm || "?"}`)
|
|
867
|
+
.sort()
|
|
868
|
+
.join(",");
|
|
869
|
+
const a = face(prev);
|
|
870
|
+
const b = face(pulse);
|
|
871
|
+
if (a && b && a !== b) {
|
|
872
|
+
notes.push({
|
|
873
|
+
kind: "listen-swap",
|
|
874
|
+
title: "На 80/443 сменился процесс",
|
|
875
|
+
text: `Было ${a}, стало ${b}. Бинарь витрины подменили или рядом встал другой сервер.`,
|
|
876
|
+
do: "Сверьте, кто должен слушать HTTPS. Orb44 процесс сам не откатывает.",
|
|
877
|
+
});
|
|
878
|
+
}
|
|
879
|
+
}
|
|
608
880
|
return notes;
|
|
609
881
|
}
|
|
610
882
|
|
|
@@ -618,7 +890,7 @@ export function insideWatchReasons(pulse, prev, rec = {}) {
|
|
|
618
890
|
const notes = [];
|
|
619
891
|
const incident = String(streetIncidentCode(rec) || "");
|
|
620
892
|
const streetHot = /auth_hot|auth_open|l7_exhaustion/.test(incident);
|
|
621
|
-
const loadHigh = Number(pulse?.load1) >= 2;
|
|
893
|
+
const loadHigh = Number(pulse?.load1) >= Math.max(2, cpuCount(pulse));
|
|
622
894
|
const memHigh = pulse?.memTotal && pulse.memUsed / pulse.memTotal >= 0.85;
|
|
623
895
|
const top = pulse?.top?.[0];
|
|
624
896
|
const who = top ? `${top.comm} ${top.cpuPct}%` : "процесс в топе не виден";
|
|
@@ -657,6 +929,52 @@ export function insideWatchReasons(pulse, prev, rec = {}) {
|
|
|
657
929
|
text: `Появились ${added.join(", ")}. С прошлого пульса их не было.`,
|
|
658
930
|
});
|
|
659
931
|
}
|
|
932
|
+
const face = (p) =>
|
|
933
|
+
(p?.listen || [])
|
|
934
|
+
.filter((r) => Number(r.port) === 80 || Number(r.port) === 443)
|
|
935
|
+
.map((r) => `${r.port}:${r.comm || "?"}`)
|
|
936
|
+
.sort()
|
|
937
|
+
.join(",");
|
|
938
|
+
const a = face(prev);
|
|
939
|
+
const b = face(pulse);
|
|
940
|
+
if (a && b && a !== b) {
|
|
941
|
+
notes.push({
|
|
942
|
+
kind: "listen-swap",
|
|
943
|
+
title: "На 80/443 сменился процесс",
|
|
944
|
+
text: `Было ${a}, стало ${b}.`,
|
|
945
|
+
});
|
|
946
|
+
}
|
|
947
|
+
}
|
|
948
|
+
if (stuckFromTop(pulse?.top).length) {
|
|
949
|
+
notes.push({
|
|
950
|
+
kind: "stuck-proc",
|
|
951
|
+
title: "Процессы зависли",
|
|
952
|
+
text: stuckFromTop(pulse.top)
|
|
953
|
+
.map((r) => `${r.comm} ${r.stat}`)
|
|
954
|
+
.join(", "),
|
|
955
|
+
});
|
|
956
|
+
}
|
|
957
|
+
if (Number(pulse?.pressure?.conntrackPct) >= 80) {
|
|
958
|
+
notes.push({
|
|
959
|
+
kind: "conntrack",
|
|
960
|
+
title: "conntrack почти полный",
|
|
961
|
+
text: `${pulse.pressure.conntrackPct}%`,
|
|
962
|
+
});
|
|
963
|
+
}
|
|
964
|
+
const overDelta =
|
|
965
|
+
pulse?.pressure?.listenOverflows != null && prev?.pressure?.listenOverflows != null
|
|
966
|
+
? Number(pulse.pressure.listenOverflows) - Number(prev.pressure.listenOverflows)
|
|
967
|
+
: 0;
|
|
968
|
+
const dropDelta =
|
|
969
|
+
pulse?.pressure?.listenDrops != null && prev?.pressure?.listenDrops != null
|
|
970
|
+
? Number(pulse.pressure.listenDrops) - Number(prev.pressure.listenDrops)
|
|
971
|
+
: 0;
|
|
972
|
+
if (overDelta > 0 || dropDelta > 0) {
|
|
973
|
+
notes.push({
|
|
974
|
+
kind: "backlog",
|
|
975
|
+
title: "Очередь accept растёт",
|
|
976
|
+
text: `+${overDelta} overflows, +${dropDelta} drops с прошлого пульса.`,
|
|
977
|
+
});
|
|
660
978
|
}
|
|
661
979
|
const streetIp = rec.ticket?.ip || rec.watch?.current?.ip || rec.watch?.last?.ip || null;
|
|
662
980
|
if (pulse?.originA?.length && streetIp && !pulse.originA.includes(streetIp)) {
|
|
@@ -684,6 +1002,7 @@ export function collectPulse() {
|
|
|
684
1002
|
const top = topTable();
|
|
685
1003
|
const hardening = collectHardening(listen);
|
|
686
1004
|
const limited = listen.length > 0 && !named;
|
|
1005
|
+
const pressure = collectPressure();
|
|
687
1006
|
return sanitizePulse({
|
|
688
1007
|
ts: Date.now(),
|
|
689
1008
|
hostname: os.hostname(),
|
|
@@ -695,9 +1014,10 @@ export function collectPulse() {
|
|
|
695
1014
|
listen,
|
|
696
1015
|
originA: originAddrs(),
|
|
697
1016
|
failedUnit: null,
|
|
698
|
-
oom: Number(hardening.oomKills) > 0,
|
|
1017
|
+
oom: Number(hardening.oomKills) > 0 || Number(pressure.cgroupOom) > 0,
|
|
699
1018
|
limited,
|
|
700
1019
|
hardening,
|
|
1020
|
+
pressure,
|
|
701
1021
|
});
|
|
702
1022
|
}
|
|
703
1023
|
|
|
@@ -766,6 +1086,13 @@ export function formatPulsePreview(pulse, lang = "en") {
|
|
|
766
1086
|
[t(lang, "preview_row_ssh"), ssh],
|
|
767
1087
|
]
|
|
768
1088
|
);
|
|
769
|
-
const
|
|
1089
|
+
const extraBits = [];
|
|
1090
|
+
if (pulse.limited) extraBits.push(t(lang, "preview_limited"));
|
|
1091
|
+
const stuck = stuckFromTop(pulse.top);
|
|
1092
|
+
if (stuck.length) extraBits.push(`D/Z/T ${stuck.map((r) => `${r.comm}:${r.stat}`).join(" ")}`);
|
|
1093
|
+
const pr = pulse.pressure || {};
|
|
1094
|
+
if (pr.conntrackPct != null) extraBits.push(`conntrack ${pr.conntrackPct}%`);
|
|
1095
|
+
if (pr.cpus) extraBits.push(`${pr.cpus} CPU`);
|
|
1096
|
+
const extra = extraBits.length ? `\n${extraBits.join(" · ")}` : "";
|
|
770
1097
|
return `${machine}\n\n${listen}\n\n${top}\n\n${guard}${extra}`;
|
|
771
1098
|
}
|