@volter/supercode-health 0.1.4 → 0.1.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/bin/health.mjs +3 -2
- package/lib/darwin.mjs +2 -0
- package/lib/format.mjs +4 -1
- package/lib/linux.mjs +10 -4
- package/lib/mail.mjs +39 -6
- package/lib/probe.mjs +43 -7
- package/lib/sample.mjs +75 -13
- package/lib/thresholds.mjs +64 -11
- package/lib/win32.mjs +3 -1
- package/package.json +1 -1
package/bin/health.mjs
CHANGED
|
@@ -43,8 +43,9 @@ if (values.help || verb === 'help') {
|
|
|
43
43
|
const { judge, raised } = await import('../lib/thresholds.mjs');
|
|
44
44
|
// A single look has no windows behind it: report what each threshold meets right now.
|
|
45
45
|
for (const o of observations) o.def = { ...o.def, forSec: 0, critForSec: 0 };
|
|
46
|
-
|
|
47
|
-
|
|
46
|
+
const unknowns = {};
|
|
47
|
+
judge(probe.state, observations, families, now, unknowns);
|
|
48
|
+
const snapshot = { version: 1, machine: probe.machine, platform: probe.platform, at: new Date(now).toISOString(), probe: { pid: process.pid, intervalSec: 15, once: true }, features: probe.features().map((x) => ({ ...x, ...(errors[x.id] ? { error: errors[x.id] } : {}) })), alarms: raised(probe.state), unknowns: Object.entries(unknowns).map(([key, u]) => ({ key, family: u.family, why: u.why, since: new Date(u.since).toISOString() })), readings };
|
|
48
49
|
(await import('node:fs')).rmSync(probe.dir, { recursive: true, force: true });
|
|
49
50
|
print(snapshot, formatStatus(snapshot, { all: values.all }));
|
|
50
51
|
} else if (verb === 'status') {
|
package/lib/darwin.mjs
CHANGED
|
@@ -142,6 +142,8 @@ export async function volumes() {
|
|
|
142
142
|
if (kept.some((k) => k.freeBytes === row.freeBytes && k.device.replace(/s\d+(s\d+)?$/, '') === row.device.replace(/s\d+(s\d+)?$/, ''))) continue;
|
|
143
143
|
kept.push(row);
|
|
144
144
|
}
|
|
145
|
+
// the listing is every mount only when df finished well: one that failed (a hung volume) listed what it could
|
|
146
|
+
kept.unread = r.ok ? null : `df did not finish (${r.error}); the volumes it did not list are unknown`;
|
|
145
147
|
return kept;
|
|
146
148
|
}
|
|
147
149
|
|
package/lib/format.mjs
CHANGED
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
// The snapshot as a person reads it: alarms first, then each reading, then which features are off and why.
|
|
2
|
+
import { groupUnknowns } from './mail.mjs';
|
|
2
3
|
const n = (v, unit = '') => (v == null ? '-' : `${v}${unit}`);
|
|
3
4
|
const who = (p) => (p?.session?.address ? ` ← ${p.session.address}` : p?.session?.harness ? ` ← ${p.session.harness} pid ${p.session.agentPid}` : p?.session?.note ? ` ← (${p.session.note})` : p?.session?.inherited ? ` ← (no agent above; inherited ${p.session.inherited})` : '');
|
|
4
5
|
|
|
@@ -9,7 +10,9 @@ export function formatStatus(s, { all = false } = {}) {
|
|
|
9
10
|
const out = [`machine health · ${s.machine} (${s.platform}) · read ${age}s ago by probe pid ${s.probe?.pid}${s.probe?.config ? ` · config ${s.probe.config}` : ''}`];
|
|
10
11
|
if (age > 3 * (s.probe?.intervalSec ?? 15)) out.push(` STALE: the probe has not written for ${age}s; its readings below are old`);
|
|
11
12
|
out.push('', s.alarms.length ? `ALARMS (${s.alarms.length})` : 'ALARMS: none');
|
|
12
|
-
for (const a of s.alarms) out.push(` ${a.level.toUpperCase().padEnd(4)} ${a.summary}${a.detail?.pid ? ` [${a.detail.command?.slice(0, 80)}${who(a.detail)}]` : ''} (since ${a.since})`);
|
|
13
|
+
for (const a of s.alarms) out.push(` ${a.level.toUpperCase().padEnd(4)} ${a.summary}${a.detail?.pid ? ` [${a.detail.command?.slice(0, 80)}${who(a.detail)}]` : ''} (since ${a.since})${a.unknown ? `\n reading unknown since ${a.unknown.since}: ${a.unknown.why}` : ''}`);
|
|
14
|
+
// what could not be read (raised or not): unknown, with since when and why
|
|
15
|
+
for (const g of groupUnknowns(s.unknowns ?? [])) out.push(` ${g.line}`);
|
|
13
16
|
if (s.maintainers) out.push(` mailed to: ${s.maintainers.length ? s.maintainers.join(', ') : 'no maintainer subscribed (supercode health subscribe <address>)'}${s.mail?.error ? ` · last mail not filed: ${s.mail.error}` : s.mail?.filed ? ` · last mail filed ${s.mail.at} (supercode message waiting lists any not yet delivered)` : ''}`);
|
|
14
17
|
if (r.load) out.push('', `load ${r.load.one} / ${r.load.five} / ${r.load.fifteen} on ${r.load.cores} cores (${r.load.perCore}× per core)`);
|
|
15
18
|
if (r.memory) {
|
package/lib/linux.mjs
CHANGED
|
@@ -11,11 +11,11 @@ export async function memory() {
|
|
|
11
11
|
const [key, value] = line.split(':'); return [key, Number.parseInt(value, 10) * 1024];
|
|
12
12
|
}));
|
|
13
13
|
const vmstat = Object.fromEntries(read('/proc/vmstat').split('\n').filter(Boolean).map((line) => line.split(' ')).map(([k, v]) => [k, Number(v)]));
|
|
14
|
-
let psi = null;
|
|
14
|
+
let psi = null, noPsi = false;
|
|
15
15
|
try {
|
|
16
16
|
const text = read('/proc/pressure/memory');
|
|
17
17
|
psi = { someAvg10: Number(/some avg10=([\d.]+)/.exec(text)?.[1]), fullAvg10: Number(/full avg10=([\d.]+)/.exec(text)?.[1]) };
|
|
18
|
-
} catch { /*
|
|
18
|
+
} catch (error) { noPsi = error.code === 'ENOENT' || error.code === 'EOPNOTSUPP'; /* a kernel without PSI, or with it off at boot (psi=0), has no pressure to read; any other failure is unknown */ }
|
|
19
19
|
const total = meminfo.MemTotal;
|
|
20
20
|
const level = psi ? (psi.fullAvg10 >= 10 ? 'critical' : psi.someAvg10 >= 10 ? 'warn' : 'normal') : null;
|
|
21
21
|
return {
|
|
@@ -24,6 +24,8 @@ export async function memory() {
|
|
|
24
24
|
swapUsedBytes: meminfo.SwapTotal - meminfo.SwapFree, swapTotalBytes: meminfo.SwapTotal,
|
|
25
25
|
counters: { swapouts: vmstat.pswpout, swapins: vmstat.pswpin, pageouts: vmstat.pgpgout, pageins: vmstat.pgpgin, compressions: null, decompressions: null, majorFaults: vmstat.pgmajfault },
|
|
26
26
|
pageSize: PAGE,
|
|
27
|
+
// what Linux has no counter for: left out of judging, never unknown
|
|
28
|
+
absent: ['memory.compressions', ...(noPsi ? ['memory.pressure'] : [])],
|
|
27
29
|
};
|
|
28
30
|
}
|
|
29
31
|
|
|
@@ -111,13 +113,17 @@ export async function files() {
|
|
|
111
113
|
|
|
112
114
|
const SKIP_FS = new Set(['proc', 'sysfs', 'tmpfs', 'devtmpfs', 'devpts', 'cgroup', 'cgroup2', 'overlay', 'squashfs', 'securityfs', 'debugfs', 'tracefs', 'pstore', 'bpf', 'mqueue', 'hugetlbfs', 'fusectl', 'configfs', 'autofs', 'binfmt_misc', 'nsfs', 'ramfs', 'efivarfs', 'rpc_pipefs', 'fuse.portal', '9p']);
|
|
113
115
|
export async function volumes() {
|
|
114
|
-
const seen = new Set(); const rows = [];
|
|
116
|
+
const seen = new Set(); const rows = []; const failed = []; const denied = [];
|
|
115
117
|
for (const line of read('/proc/mounts').split('\n')) {
|
|
116
118
|
const [device, mount, type] = line.split(' ');
|
|
117
119
|
if (!mount || SKIP_FS.has(type) || seen.has(device)) continue;
|
|
118
120
|
seen.add(device);
|
|
119
|
-
try { const s = statfsSync(mount.replace(/\\040/g, ' ')); rows.push({ mount, device, totalBytes: s.blocks * s.bsize, freeBytes: s.bavail * s.bsize }); } catch {
|
|
121
|
+
try { const s = statfsSync(mount.replace(/\\040/g, ' ')); rows.push({ mount, device, totalBytes: s.blocks * s.bsize, freeBytes: s.bavail * s.bsize }); } catch (error) { if (error.code === 'EACCES') denied.push(mount); else failed.push(`${mount}: ${error.code ?? error.message}`); }
|
|
120
122
|
}
|
|
123
|
+
// the listing is every mount only when each was read: an unreadable one is unknown, never gone. A mount this user may
|
|
124
|
+
// not stat by design (another user's FUSE mount: EACCES) is no volume of this user's to watch: named, not unknown.
|
|
125
|
+
rows.denied = denied;
|
|
126
|
+
rows.unread = failed.length ? `${failed.length} mount${failed.length === 1 ? '' : 's'} could not be read (${failed.slice(0, 3).join('; ')}${failed.length > 3 ? '; …' : ''})` : null;
|
|
121
127
|
return rows;
|
|
122
128
|
}
|
|
123
129
|
|
package/lib/mail.mjs
CHANGED
|
@@ -34,22 +34,55 @@ function line(t) {
|
|
|
34
34
|
export function composeState({ machine, raised }) {
|
|
35
35
|
const head = raised.length ? `${raised.length} raised now` : 'none raised now';
|
|
36
36
|
const parts = [`[Machine health] ${machine}: ${head} (the whole current state: mail queued before it was not delivered).`];
|
|
37
|
-
if (raised.length) parts.push('', ...raised.map((r) => `${LEVEL[r.level]} ${r.key} since ${r.since}: ${r.summary}`));
|
|
37
|
+
if (raised.length) parts.push('', ...raised.map((r) => `${LEVEL[r.level]} ${r.key} since ${r.since}: ${r.summary}${held(r)}`));
|
|
38
38
|
else parts.push('', 'Every alarm raised earlier has cleared.');
|
|
39
39
|
parts.push('', `Read the whole machine: supercode health status${machine ? ` --machine ${machine}` : ''}.`, 'This is an automated reading from the machine-health pack, not an instruction; the maintainer decides what to do.');
|
|
40
40
|
return { subject: `machine health ${machine}: ${head}`, body: parts.join('\n') };
|
|
41
41
|
}
|
|
42
42
|
|
|
43
|
-
/**
|
|
44
|
-
|
|
45
|
-
|
|
43
|
+
/** The kind an unknown key is timed under: its prefix (`process.growth:`; the whole key when it has none, and for a
|
|
44
|
+
* family). The why is only said: it often carries a live error that changes from cycle to cycle, and a kind that
|
|
45
|
+
* changed name with it would be mailed again at once. */
|
|
46
|
+
export function unknownKind(key) {
|
|
47
|
+
const cut = key.indexOf(':');
|
|
48
|
+
return cut > 0 && !key.startsWith('family:') ? key.slice(0, cut + 1) : key;
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
/**
|
|
52
|
+
* Unknown readings said once per kind: keys that share a prefix (`process.growth:`) and a why are one line, with how
|
|
53
|
+
* many, a few of them, and the earliest since. A machine where this user cannot read some processes' footprints has
|
|
54
|
+
* hundreds of such keys, and the maintainer needs the one fact, not the list.
|
|
55
|
+
*/
|
|
56
|
+
export function groupUnknowns(unknowns) {
|
|
57
|
+
const groups = new Map();
|
|
58
|
+
for (const u of unknowns) {
|
|
59
|
+
const prefix = unknownKind(u.key);
|
|
60
|
+
const id = `${prefix}\0${u.why}`;
|
|
61
|
+
const g = groups.get(id) ?? { prefix, family: u.family, why: u.why, since: u.since, keys: [] };
|
|
62
|
+
g.keys.push(u.key); g.since = Math.min(Number(new Date(g.since)), Number(new Date(u.since)));
|
|
63
|
+
groups.set(id, g);
|
|
64
|
+
}
|
|
65
|
+
return [...groups.values()].map((g) => ({ ...g, line: `UNKNOWN ${g.keys.length === 1 ? g.keys[0] : `${g.prefix}* (${g.keys.length}: ${g.keys.slice(0, 3).map((k) => k.slice(g.prefix.length)).join(', ')}${g.keys.length > 3 ? ', …' : ''})`}${g.family ? ` (${g.family})` : ''} since ${new Date(g.since).toISOString()}: ${g.why}` }));
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
const held = (r) => r.unknown ? ` (its reading unknown since ${r.unknown.since}: ${r.unknown.why})` : '';
|
|
69
|
+
|
|
70
|
+
/**
|
|
71
|
+
* One mail for a cycle's transitions, the reminders due, and the readings unknown long enough to say (each with why
|
|
72
|
+
* and since: a reading that cannot be taken is a failure that reaches the maintainer, raised or not; invariant 7);
|
|
73
|
+
* null when there is nothing to say.
|
|
74
|
+
*/
|
|
75
|
+
export function compose({ machine, transitions, reminders, raised, unknowns = [] }) {
|
|
76
|
+
if (!transitions.length && !reminders.length && !unknowns.length) return null;
|
|
46
77
|
const up = transitions.filter((t) => t.to !== 'none');
|
|
47
78
|
const down = transitions.filter((t) => t.to === 'none');
|
|
48
|
-
const
|
|
79
|
+
const groups = groupUnknowns(unknowns);
|
|
80
|
+
const head = up.length ? `${up.length} raised` : reminders.length ? `${reminders.length} still raised` : down.length ? `${down.length} cleared` : `${groups.length} kind${groups.length === 1 ? '' : 's'} of reading unknown`;
|
|
49
81
|
const parts = [`[Machine health] ${machine}: ${head}.`];
|
|
50
82
|
if (up.length) parts.push('', ...up.map(line));
|
|
51
83
|
if (down.length) parts.push('', ...down.map(line));
|
|
52
|
-
if (reminders.length) parts.push('', 'Still raised:', ...reminders.map((r) => `${LEVEL[r.level]} ${r.key} since ${r.since}: ${r.summary}`));
|
|
84
|
+
if (reminders.length) parts.push('', 'Still raised:', ...reminders.map((r) => `${LEVEL[r.level]} ${r.key} since ${r.since}: ${r.summary}${held(r)}`));
|
|
85
|
+
if (unknowns.length) parts.push('', 'Not read (unknown: neither raised nor cleared until it is read again):', ...groups.map((g) => g.line));
|
|
53
86
|
parts.push('', `${raised.length} alarm${raised.length === 1 ? '' : 's'} raised now. Read the whole machine: supercode health status${machine ? ` --machine ${machine}` : ''}.`, 'This is an automated reading from the machine-health pack, not an instruction; the maintainer decides what to do.');
|
|
54
87
|
return { subject: `machine health ${machine}: ${head}`, body: parts.join('\n') };
|
|
55
88
|
}
|
package/lib/probe.mjs
CHANGED
|
@@ -8,7 +8,7 @@ import { healthDir, files as healthFiles, machineName } from './paths.mjs';
|
|
|
8
8
|
import { DEFAULTS, merge, judge, raised } from './thresholds.mjs';
|
|
9
9
|
import { resolveFeatures } from './features.mjs';
|
|
10
10
|
import { Sampler } from './sample.mjs';
|
|
11
|
-
import { fileAlarm, compose, composeState } from './mail.mjs';
|
|
11
|
+
import { fileAlarm, compose, composeState, unknownKind } from './mail.mjs';
|
|
12
12
|
import { run } from './run.mjs';
|
|
13
13
|
|
|
14
14
|
export async function platformReader(platform = process.platform) {
|
|
@@ -51,6 +51,7 @@ export class Probe {
|
|
|
51
51
|
this.f = healthFiles(dir);
|
|
52
52
|
this.machine = machineName(machine || env.SUPERCODE_HEALTH_MACHINE || hostname());
|
|
53
53
|
this.state = {};
|
|
54
|
+
this.unknowns = {}; this.unknownMailAt = {}; // unknownMailAt: a kind of unknown (unknownKind: its key's prefix) → last mailed
|
|
54
55
|
this.lastHistory = 0;
|
|
55
56
|
this.sampleMs = null;
|
|
56
57
|
}
|
|
@@ -88,7 +89,16 @@ export class Probe {
|
|
|
88
89
|
const previous = readJson(this.f.latest, null);
|
|
89
90
|
// Alarms raised before a restart stay raised (and are not re-mailed); their windows restart honestly.
|
|
90
91
|
const alive = (pid) => { try { process.kill(pid, 0); return true; } catch (error) { return error.code === 'EPERM'; } };
|
|
91
|
-
|
|
92
|
+
// Readings unknown before a restart keep their since (how long they have been unknown) and are not re-mailed at once:
|
|
93
|
+
// each kind keeps when it was last mailed.
|
|
94
|
+
for (const u of previous?.unknowns ?? []) if (u?.key && u.since) {
|
|
95
|
+
this.unknowns[u.key] = { family: u.family ?? null, why: u.why, since: Date.parse(u.since) };
|
|
96
|
+
const kind = unknownKind(u.key);
|
|
97
|
+
if (u.mailed) this.unknownMailAt[kind] = Math.max(this.unknownMailAt[kind] ?? 0, Date.parse(u.mailed));
|
|
98
|
+
}
|
|
99
|
+
// A raised alarm whose reading was unknown keeps its unknown since across the restart, as its unknowns entry does.
|
|
100
|
+
for (const a of (previous?.alarms ?? []).filter((x) => !x.detail?.pid || alive(x.detail.pid))) this.state[a.key] = { key: a.key, family: a.family, level: a.level, warnSince: null, critSince: null, raisedAt: a.since ? Date.parse(a.since) : Date.now(), lastMailAt: this.droppedOutbox ? 0 : Date.now(), restoredUntil: Date.now() + 2 * 60_000, value: a.value, unit: a.unit, summary: a.summary, detail: a.detail,
|
|
101
|
+
...(a.unknown?.since ? { unread: a.unknown.why, unreadAt: Date.parse(a.unknown.since) } : {}) };
|
|
92
102
|
}
|
|
93
103
|
|
|
94
104
|
/** Remove an earlier pack's outbox; answers how many mails it held, logged by recipient. */
|
|
@@ -123,16 +133,40 @@ export class Probe {
|
|
|
123
133
|
const t0 = Date.now();
|
|
124
134
|
const { readings, observations, families, errors } = await this.sampler.sample(features, now);
|
|
125
135
|
this.sampleMs = Date.now() - t0;
|
|
126
|
-
const transitions = judge(this.state, observations, families, now);
|
|
136
|
+
const transitions = judge(this.state, observations, families, now, this.unknowns);
|
|
127
137
|
const alarms = raised(this.state);
|
|
128
138
|
const repeatMs = this.config.repeatMin * 60_000;
|
|
129
|
-
|
|
139
|
+
// A raised alarm is reminded each repeatMin after it was last said: in a mail of its own transition or reminder, never
|
|
140
|
+
// pushed back by a mail about something else.
|
|
141
|
+
const raisedNow = new Set(transitions.filter((t) => t.to !== 'none').map((t) => t.key));
|
|
142
|
+
const reminders = alarms.filter((a) => !raisedNow.has(a.key) && now - (this.state[a.key].lastMailAt ?? 0) >= repeatMs);
|
|
143
|
+
// A reading unknown for unknownMin is the maintainers' to know (raised or not), and again each repeatMin until read.
|
|
144
|
+
// Timed per kind (a key's prefix; a family's whole key): a kind is due once its earliest key has been unknown
|
|
145
|
+
// unknownMin and the kind was last mailed repeatMin ago, and then every key of it is said. Keys that join a kind one
|
|
146
|
+
// by one (process growth, pid by pid) never make a mail each, and a why that changes (a live error) is only said:
|
|
147
|
+
// it never makes the kind new.
|
|
148
|
+
const unknownMs = (this.config.unknownMin ?? 15) * 60_000;
|
|
149
|
+
const kinds = new Map();
|
|
150
|
+
for (const [key, u] of Object.entries(this.unknowns)) {
|
|
151
|
+
const kind = unknownKind(key);
|
|
152
|
+
if (!kinds.has(kind)) kinds.set(kind, []);
|
|
153
|
+
kinds.get(kind).push({ key, ...u });
|
|
154
|
+
}
|
|
155
|
+
for (const kind of Object.keys(this.unknownMailAt)) if (!kinds.has(kind)) delete this.unknownMailAt[kind];
|
|
156
|
+
const dueKinds = [...kinds].filter(([kind, list]) => now - Math.min(...list.map((u) => u.since)) >= unknownMs && now - (this.unknownMailAt[kind] ?? 0) >= repeatMs);
|
|
157
|
+
// a due kind lists only its keys unknown unknownMin or longer: a younger one is said once it is that old
|
|
158
|
+
const unknownsDue = dueKinds.flatMap(([, list]) => list.filter((u) => now - u.since >= unknownMs));
|
|
130
159
|
const to = subscribers(this.f, this.config);
|
|
131
160
|
// after dropping an earlier pack's undelivered mail, the first mail states the whole current state, clears included
|
|
132
|
-
const
|
|
133
|
-
|
|
161
|
+
const composedState = Boolean(this.droppedOutbox && to.length);
|
|
162
|
+
const mail = composedState ? composeState({ machine: this.machine, raised: alarms }) : compose({ machine: this.machine, transitions, reminders, raised: alarms, unknowns: unknownsDue });
|
|
163
|
+
if (composedState) this.droppedOutbox = 0;
|
|
134
164
|
if (mail) {
|
|
135
|
-
|
|
165
|
+
// the kinds of unknown this mail said (the whole-state mail says none: they stay due)
|
|
166
|
+
if (!composedState) for (const [kind] of dueKinds) this.unknownMailAt[kind] = now;
|
|
167
|
+
// only the alarms this mail said: raised by it, or reminded (the whole-state mail says every one)
|
|
168
|
+
const said = composedState ? alarms : alarms.filter((a) => raisedNow.has(a.key) || reminders.includes(a));
|
|
169
|
+
for (const a of said) this.state[a.key].lastMailAt = now;
|
|
136
170
|
this.log(`${mail.subject}${to.length ? '' : ' (no maintainer subscribed)'}\n${mail.body}`);
|
|
137
171
|
}
|
|
138
172
|
const snapshot = {
|
|
@@ -140,6 +174,8 @@ export class Probe {
|
|
|
140
174
|
probe: { pid: process.pid, intervalSec: this.config.intervalSec, slowIntervalSec: this.config.slowIntervalSec, sampleMs: this.sampleMs, user: this.env.USER ?? null, root: this.reader.isRoot?.() ?? false, config: this.config.source },
|
|
141
175
|
features: features.map(({ id, on, barrier, reason, reads }) => ({ id, on, barrier, reason, reads, ...(errors[id] ? { error: errors[id] } : {}) })),
|
|
142
176
|
alarms,
|
|
177
|
+
// what could not be read, raised or not, with since when and why (kept across a restart)
|
|
178
|
+
unknowns: Object.entries(this.unknowns).map(([key, u]) => { const mailed = this.unknownMailAt[unknownKind(key)]; return { key, family: u.family, why: u.why, since: new Date(u.since).toISOString(), mailed: mailed ? new Date(mailed).toISOString() : null }; }),
|
|
143
179
|
transitions: transitions.map((t) => ({ key: t.key, from: t.from, to: t.to, summary: t.summary })),
|
|
144
180
|
maintainers: to,
|
|
145
181
|
mail: this.lastMail ?? null,
|
package/lib/sample.mjs
CHANGED
|
@@ -27,6 +27,14 @@ function argvSession(command) {
|
|
|
27
27
|
const SLOW_FEATURE = { disk: 'disk', services: 'services', service: 'services', supercode: 'supercode', browserTabs: 'browser-tabs', tabs: 'browser-tabs', power: 'power' };
|
|
28
28
|
// Every alarm family, by the feature that observes it.
|
|
29
29
|
const FAMILY_FEATURE = { load: 'load', memory: 'memory', process: 'processes', browser: 'renderers', daemon: 'daemons', 'disk-writer': 'disk-io', files: 'files', tcp: 'tcp', disk: 'disk', service: 'services', supercode: 'supercode', tabs: 'browser-tabs', power: 'power' };
|
|
30
|
+
// The families a feature's read observes: when its read fails, each is unread (judged neither raised nor cleared).
|
|
31
|
+
// `processes` also observes the renderer, daemon and disk-writer families it reads beside its own; `disk-io`'s own read
|
|
32
|
+
// (device throughput) judges no family: the disk writers are the process read's, so a throughput read that fails leaves
|
|
33
|
+
// them as that read found them.
|
|
34
|
+
const FEATURE_FAMILIES = { processes: ['process', 'browser', 'daemon', 'disk-writer'], 'disk-io': [] };
|
|
35
|
+
const familiesOf = (feature) => FEATURE_FAMILIES[feature] ?? Object.entries(FAMILY_FEATURE).filter(([, f]) => f === feature).map(([family]) => family);
|
|
36
|
+
/** What a failed read leaves unknown, said to the judge (thresholds.mjs judge): never left out, which would clear. */
|
|
37
|
+
const unreadFamilies = (feature, why) => familiesOf(feature).map((family) => ({ unreadFamily: family, family, why: `${feature} could not be read: ${why}` }));
|
|
30
38
|
|
|
31
39
|
export class Sampler {
|
|
32
40
|
constructor({ reader, platform = process.platform, machine, env = process.env, config }) {
|
|
@@ -56,7 +64,7 @@ export class Sampler {
|
|
|
56
64
|
const errors = {};
|
|
57
65
|
const dt = this.prev ? (now - this.prev.at) / 1000 : null;
|
|
58
66
|
const attempt = async (id, fn) => {
|
|
59
|
-
try { await fn(); } catch (error) { errors[id] = error.message; }
|
|
67
|
+
try { await fn(); } catch (error) { errors[id] = error.message; observations.push(...unreadFamilies(id, error.message)); }
|
|
60
68
|
};
|
|
61
69
|
const alarm = (name) => cfg.alarms[name];
|
|
62
70
|
|
|
@@ -88,11 +96,16 @@ export class Sampler {
|
|
|
88
96
|
...(memory.psi ? { psi: memory.psi } : {}),
|
|
89
97
|
};
|
|
90
98
|
families.add('memory');
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
if (
|
|
99
|
+
// A metric this platform does not have (memory.absent) is left out of judging: never raised, never unknown. Any
|
|
100
|
+
// other the platform gave no value for this cycle is unknown (null), never left out: left out, it would clear.
|
|
101
|
+
const absent = new Set(memory.absent ?? []);
|
|
102
|
+
const judged = (o) => { if (!absent.has(o.key ?? o.unreadKey)) observations.push(o); };
|
|
103
|
+
if (!memory.pressureLevel) judged({ unreadKey: 'memory.pressure', family: 'memory', why: 'the kernel memory pressure was not read this cycle' });
|
|
104
|
+
else judged({ key: 'memory.pressure', family: 'memory', def: alarm('memory.pressure'), value: memory.pressureLevel, met: { warn: memory.pressureLevel !== 'normal', crit: memory.pressureLevel === 'critical', clears: memory.pressureLevel === 'normal' }, summary: `kernel memory pressure ${memory.pressureLevel}` });
|
|
105
|
+
judged({ key: 'memory.free', family: 'memory', def: alarm('memory.free'), value: memory.freePct, unit: '%', summary: `memory free ${memory.freePct}%` });
|
|
106
|
+
judged({ key: 'memory.swapping', family: 'memory', def: alarm('memory.swapping'), value: rate('swapouts'), unit: 'pages/s', summary: `swapping out ${round(rate('swapouts'))} pages/s` });
|
|
107
|
+
judged({ key: 'memory.swapped30m', family: 'memory', def: alarm('memory.swapped30m'), value: swapped30, unit: '% of RAM', summary: `${round(swapped30, 2)}% of RAM swapped out in the last ${readings.memory.swappedWindowMin} min` });
|
|
108
|
+
judged({ key: 'memory.compressions', family: 'memory', def: alarm('memory.compressions'), value: rate('compressions'), unit: 'pages/s', summary: `${round(rate('compressions'), 0)} compressions/s` });
|
|
96
109
|
});
|
|
97
110
|
|
|
98
111
|
// ---- processes, renderers, daemons, writers ------------------------------------------------------
|
|
@@ -100,7 +113,12 @@ export class Sampler {
|
|
|
100
113
|
if (on.has('processes')) await attempt('processes', async () => {
|
|
101
114
|
const [rows, rus] = await Promise.all([
|
|
102
115
|
this.reader.processes(),
|
|
103
|
-
on.has('process-footprint') ? this.reader.rusage().catch((error) => {
|
|
116
|
+
on.has('process-footprint') ? this.reader.rusage().catch((error) => {
|
|
117
|
+
errors['process-footprint'] = error.message;
|
|
118
|
+
// without footprints no growth is read: every growth alarm is unknown this cycle, not cleared
|
|
119
|
+
for (const prefix of ['process.growth:', 'renderer.growth:']) observations.push({ unreadPrefix: prefix, family: 'process', why: `process footprints could not be read: ${error.message}` });
|
|
120
|
+
return new Map();
|
|
121
|
+
}) : new Map(),
|
|
104
122
|
]);
|
|
105
123
|
if (this.platform === 'darwin' && now - this.topFoot.at >= cfg.footprintTopEverySec * 1000) {
|
|
106
124
|
this.topFoot = { at: now, map: await this.reader.topFootprints().catch((error) => { errors.daemons = error.message; return this.topFoot.map; }) };
|
|
@@ -145,6 +163,14 @@ export class Sampler {
|
|
|
145
163
|
families.add('process');
|
|
146
164
|
const daemonNames = new Set(cfg.daemons?.[this.platform] ?? []);
|
|
147
165
|
const isDaemon = (r) => daemonNames.has(base(r.command));
|
|
166
|
+
// What this read covers, so a raised alarm it no longer shows clears only on that (thresholds.mjs judge): the
|
|
167
|
+
// process list is complete (a pid not in it is gone), and each check names the processes it judged on a value.
|
|
168
|
+
const pids = new Set(rows.map((r) => r.pid));
|
|
169
|
+
observations.push({ enumerated: 'process', family: 'process', by: 'pid', entities: pids });
|
|
170
|
+
const cpuJudged = (renderer) => new Set(byCpu.filter((r) => !isDaemon(r) && !AGENT.test(base(r.command)) && RENDERER.test(r.command) === renderer).map((r) => r.pid));
|
|
171
|
+
observations.push({ covered: 'process.cpu:', family: 'process', entities: cpuJudged(false) }, { covered: 'renderer.cpu:', family: 'process', entities: cpuJudged(true) });
|
|
172
|
+
const grown = (renderer) => new Set(rows.filter((r) => r.growthMBPerMin != null && RENDERER.test(r.command) === renderer).map((r) => r.pid));
|
|
173
|
+
observations.push({ covered: 'process.growth:', family: 'process', entities: grown(false) }, { covered: 'renderer.growth:', family: 'process', entities: grown(true) });
|
|
148
174
|
// Every process down to the lower clear line is observed, so a raised alarm holds through a brief dip
|
|
149
175
|
// instead of clearing and raising again (hysteresis); one that falls below it is no longer watched.
|
|
150
176
|
for (const r of byCpu) {
|
|
@@ -159,8 +185,9 @@ export class Sampler {
|
|
|
159
185
|
// observed; a raised one holds until its growth falls under a quarter of the line, so a leak that pauses
|
|
160
186
|
// between samples is one alarm, not a raise and a clear each minute.
|
|
161
187
|
for (const r of rows) {
|
|
162
|
-
if (r.growthMBPerMin == null) continue;
|
|
163
188
|
const renderer = RENDERER.test(r.command);
|
|
189
|
+
// a process whose growth has no reading this cycle (no footprint, or no earlier one) is unknown, not cleared
|
|
190
|
+
if (r.growthMBPerMin == null) { observations.push({ unreadKey: `${renderer ? 'renderer' : 'process'}.growth:${r.pid}`, family: 'process', why: 'no footprint reading to measure its growth this cycle' }); continue; }
|
|
164
191
|
const tab = renderer ? this.rendererTabs?.get(r.pid) ?? null : null;
|
|
165
192
|
const def = alarm(renderer ? 'renderer.growth' : 'process.growth');
|
|
166
193
|
if (r.footprintBytes / MB < def.minMB) continue;
|
|
@@ -180,6 +207,7 @@ export class Sampler {
|
|
|
180
207
|
orphanedHeadless: orphaned.map(view),
|
|
181
208
|
};
|
|
182
209
|
families.add('browser');
|
|
210
|
+
observations.push({ enumerated: 'browser', family: 'browser', by: 'pid', entities: pids }, { covered: 'browser.orphan:', family: 'browser', entities: pids });
|
|
183
211
|
for (const r of orphaned) observations.push({ key: `browser.orphan:${r.pid}`, family: 'browser', def: alarm('browser.orphan'), value: 1, met: { warn: true, crit: false, clears: false }, summary: `headless browser ${r.pid} left running with launchd as its parent`, detail: view(r) });
|
|
184
212
|
}
|
|
185
213
|
|
|
@@ -187,6 +215,8 @@ export class Sampler {
|
|
|
187
215
|
const daemons = rows.filter(isDaemon);
|
|
188
216
|
readings.daemons = daemons.sort((a, b) => (b.cpuPct ?? 0) - (a.cpuPct ?? 0)).map((r) => ({ name: base(r.command), ...view(r) }));
|
|
189
217
|
families.add('daemon');
|
|
218
|
+
observations.push({ enumerated: 'daemon', family: 'daemon', by: 'pid', entities: pids },
|
|
219
|
+
{ covered: 'daemon.cpu:', family: 'daemon', entities: new Set(daemons.filter((r) => r.cpuPct != null).map((r) => `${base(r.command)}:${r.pid}`)) });
|
|
190
220
|
const footDef = alarm('daemon.footprint');
|
|
191
221
|
for (const r of daemons) {
|
|
192
222
|
const name = base(r.command);
|
|
@@ -200,6 +230,7 @@ export class Sampler {
|
|
|
200
230
|
const writers = rows.filter((r) => r.writeBps > 0).sort((a, b) => b.writeBps - a.writeBps).slice(0, 5);
|
|
201
231
|
readings.diskWriters = writers.map((r) => ({ ...view(r), writeMBps: round(r.writeBps / MB, 1) }));
|
|
202
232
|
families.add('disk-writer');
|
|
233
|
+
observations.push({ enumerated: 'disk-writer', family: 'disk-writer', by: 'pid', entities: pids }, { covered: 'disk.writer:', family: 'disk-writer', entities: new Set(rows.filter((r) => r.writeBps != null).map((r) => r.pid)) });
|
|
203
234
|
for (const r of rows) if (r.writeBps != null && r.writeBps / MB > alarm('disk.writer').warn * 0.5) observations.push({ key: `disk.writer:${r.pid}`, family: 'disk-writer', def: alarm('disk.writer'), value: r.writeBps / MB, unit: 'MB/s', summary: `process ${r.pid} writing ${round(r.writeBps / MB, 1)} MB/s`, detail: view(r) });
|
|
204
235
|
}
|
|
205
236
|
|
|
@@ -276,6 +307,8 @@ export class Sampler {
|
|
|
276
307
|
});
|
|
277
308
|
readings.tcp = { states, timeWait: states.TIME_WAIT ?? 0, range: { ...range, size }, timeWaitSec, ephemeral, timeWaitByTarget };
|
|
278
309
|
families.add('tcp');
|
|
310
|
+
// every socket was read: an address or a target with none has none in use (judged, under its line)
|
|
311
|
+
observations.push({ covered: 'tcp.ephemeral:', family: 'tcp', all: true }, { covered: 'tcp.timeWaitTarget:', family: 'tcp', all: true });
|
|
279
312
|
for (const e of ephemeral) observations.push({ key: `tcp.ephemeral:${e.addr}`, family: 'tcp', def: alarm('tcp.ephemeral'), value: e.pct, unit: '% of ports', summary: `${e.used} of ${size} ephemeral ports in use on ${e.addr}` });
|
|
280
313
|
for (const t of timeWaitByTarget) observations.push({ key: `tcp.timeWaitTarget:${t.target}`, family: 'tcp', def, value: t.pct, unit: '% of ports', summary: `${t.count} TIME_WAIT against ${t.target.startsWith(':') ? `listener ${t.target}${t.exited ? ' (listener has exited; its TIME_WAITs are draining)' : ''}` : t.target}${t.listener ? ` (pid ${t.listener.pid})` : ''}; range ${size}, TIME_WAIT lasts ${timeWaitSec} s`, detail: t });
|
|
281
314
|
});
|
|
@@ -295,8 +328,8 @@ export class Sampler {
|
|
|
295
328
|
for (const o of this.slow.observations) if (on.has(SLOW_FEATURE[o.family])) observations.push(o);
|
|
296
329
|
for (const f of this.slow.families ?? []) if (on.has(SLOW_FEATURE[f])) families.add(f);
|
|
297
330
|
|
|
298
|
-
// A feature turned off clears its raised alarms: its family is
|
|
299
|
-
for (const [family, feature] of Object.entries(FAMILY_FEATURE)) if (!on.has(feature))
|
|
331
|
+
// A feature turned off clears its raised alarms: a positive observation that its family is no longer watched.
|
|
332
|
+
for (const [family, feature] of Object.entries(FAMILY_FEATURE)) if (!on.has(feature)) observations.push({ off: family, family });
|
|
300
333
|
|
|
301
334
|
await this.#sessions(readings, observations);
|
|
302
335
|
this.prev = { at: now, memory, io: this.io ?? null };
|
|
@@ -306,13 +339,25 @@ export class Sampler {
|
|
|
306
339
|
async #slow(on, slow, errors, now) {
|
|
307
340
|
const cfg = this.config;
|
|
308
341
|
const alarm = (name) => cfg.alarms[name];
|
|
309
|
-
const attempt = async (id, fn) => { try { await fn(); } catch (error) { errors[id] = error.message; } };
|
|
342
|
+
const attempt = async (id, fn) => { try { await fn(); } catch (error) { errors[id] = error.message; slow.observations.push(...unreadFamilies(id, error.message)); } };
|
|
310
343
|
|
|
311
344
|
if (on.has('disk')) await attempt('disk', async () => {
|
|
312
345
|
const def = alarm('disk.free');
|
|
313
346
|
const fill = alarm('disk.filling');
|
|
314
347
|
const vols = await this.reader.volumes();
|
|
315
348
|
slow.families.add('disk');
|
|
349
|
+
// Every mount was read only when the listing finished and read each one: then one not listed is gone. A listing
|
|
350
|
+
// that did not (df failed, a mount's statfs failed) enumerates nothing, so a hung volume's alarm holds, unknown.
|
|
351
|
+
// Mounts this user may not stat (EACCES): named in the reading. They still exist, so they stay in the enumeration
|
|
352
|
+
// (never gone): an alarm raised on one holds, unknown, with why; one never raised costs nothing.
|
|
353
|
+
const denied = vols.denied ?? [];
|
|
354
|
+
if (denied.length) slow.readings.diskDenied = denied;
|
|
355
|
+
for (const mount of denied) for (const prefix of ['disk.free:', 'disk.filling:']) slow.observations.push({ unreadPrefix: `${prefix}${mount}`, family: 'disk', why: `${mount} can no longer be read by this user (statfs: EACCES)` });
|
|
356
|
+
if (vols.unread) for (const prefix of ['disk.free:', 'disk.filling:']) slow.observations.push({ unreadPrefix: prefix, family: 'disk', why: vols.unread });
|
|
357
|
+
else slow.observations.push({ enumerated: 'disk', family: 'disk', by: 'suffix', entities: new Set([...vols.map((v) => v.mount), ...denied]) });
|
|
358
|
+
// a mount's filling is judged once it has a rate (10 min of readings)
|
|
359
|
+
const rated = new Set();
|
|
360
|
+
slow.observations.push({ covered: 'disk.filling:', family: 'disk', entities: rated });
|
|
316
361
|
slow.readings.disk = vols.map((v) => {
|
|
317
362
|
const ring = this.diskRing.get(v.mount) ?? [];
|
|
318
363
|
ring.push([now, v.freeBytes]);
|
|
@@ -322,6 +367,7 @@ export class Sampler {
|
|
|
322
367
|
const hours = (now - t0) / 3_600_000;
|
|
323
368
|
const perHour = hours >= 10 / 60 ? (f0 - v.freeBytes) / hours : null; // bytes consumed per hour
|
|
324
369
|
const hoursToFull = perHour > 0 ? v.freeBytes / perHour : null;
|
|
370
|
+
if (perHour != null) rated.add(v.mount);
|
|
325
371
|
const freePct = (v.freeBytes / v.totalBytes) * 100;
|
|
326
372
|
const per = cfg.volumes?.[v.mount] ?? {};
|
|
327
373
|
const row = { mount: v.mount, freeGB: round(v.freeBytes / GB), totalGB: round(v.totalBytes / GB), freePct: round(freePct), fillGBPerHour: perHour == null ? null : round(perHour / GB, 2), hoursToFull: round(hoursToFull) };
|
|
@@ -338,6 +384,8 @@ export class Sampler {
|
|
|
338
384
|
const list = cfg.services ?? [];
|
|
339
385
|
const jobs = await this.reader.services(list.map((s) => s.label));
|
|
340
386
|
slow.families.add('service');
|
|
387
|
+
// every configured service was read (one no longer configured is gone)
|
|
388
|
+
slow.observations.push({ enumerated: 'service', family: 'service', by: 'suffix', entities: new Set(list.map((s) => s.label)) });
|
|
341
389
|
slow.readings.services = list.map((s) => {
|
|
342
390
|
const job = jobs.get(s.label);
|
|
343
391
|
let logAgeMin = null;
|
|
@@ -393,7 +441,17 @@ export class Sampler {
|
|
|
393
441
|
const cdef = alarm('supercode.connector');
|
|
394
442
|
const link = connector?.enrolled ? (connector.running ? (connection == null ? null : connection === 'online') : false) : true;
|
|
395
443
|
slow.observations.push({ key: 'supercode.daemon', family: 'supercode', def: cdef, value: described ? 0 : 1, met: { warn: !described, crit: false, clears: Boolean(described) }, summary: described ? 'machine daemon answering' : `machine daemon did not answer describe (${describe.error || 'no answer'})` });
|
|
396
|
-
|
|
444
|
+
// the link is read from two sources (the connector's status, the server's machine list): either unreadable leaves it
|
|
445
|
+
// unknown, which holds a raised alarm and clears nothing (a raised link alarm once cleared while the link was down)
|
|
446
|
+
if (!data) slow.observations.push({ unreadKey: 'supercode.link', family: 'supercode', why: `the connector's status could not be read: ${status.error || 'no answer'}` });
|
|
447
|
+
else if (connector && link === null) slow.observations.push({ unreadKey: 'supercode.link', family: 'supercode', why: `the server's view of this machine could not be read: ${machines.ok === false ? machines.error || 'no answer' : listed ? 'its machine record names no connection' : 'this machine is not in the machine list'}` });
|
|
448
|
+
// a status that names no connector says nothing of the link: unknown, said, never left out
|
|
449
|
+
else if (!connector) slow.observations.push({ unreadKey: 'supercode.link', family: 'supercode', why: 'the connector\'s status names no connector' });
|
|
450
|
+
else slow.observations.push({ key: 'supercode.link', family: 'supercode', def: cdef, value: link ? 0 : 1, met: { warn: !link, crit: false, clears: link }, summary: link ? 'connector linked to its server' : `connector ${connector.running ? `running but the server sees this machine ${connection}` : 'not running'}` });
|
|
451
|
+
// launches the daemon could not list are unknown: a raised launch alarm holds
|
|
452
|
+
if (!Array.isArray(rows)) for (const prefix of ['supercode.launch:', 'supercode.launchFailed:']) slow.observations.push({ unreadPrefix: prefix, family: 'supercode', why: `the daemon's launches could not be listed: ${launches.error || 'no answer'}` });
|
|
453
|
+
// the launch list was read: every launch in it was judged stuck or not, failed or not (one not shown is neither)
|
|
454
|
+
if (Array.isArray(rows)) slow.observations.push({ covered: 'supercode.launch:', family: 'supercode', all: true }, { covered: 'supercode.launchFailed:', family: 'supercode', all: true });
|
|
397
455
|
for (const r of stuck) slow.observations.push({ key: `supercode.launch:${r.key}`, family: 'supercode', def: { forSec: 0 }, value: 1, met: { warn: true, crit: false, clears: false }, summary: r.status === 'issued' ? `launch ${r.key} (${r.kind ?? 'pane'}) issued ${Math.round((now - r.at) / 60_000)} min ago and never became ready` : `launch ${r.key} is live but its first input was not delivered: ${r.inputError}` });
|
|
398
456
|
for (const r of failed) slow.observations.push({ key: `supercode.launchFailed:${r.key}`, family: 'supercode', def: { forSec: 0 }, value: 1, met: { warn: true, crit: false, clears: false }, summary: `launch ${r.key} (${r.kind ?? 'pane'}) failed at ${new Date(r.at).toISOString()}` });
|
|
399
457
|
});
|
|
@@ -428,7 +486,10 @@ export class Sampler {
|
|
|
428
486
|
slow.readings.browserTabs = { count: tabs.length, byHolder, tabs: rows };
|
|
429
487
|
slow.families.add('tabs');
|
|
430
488
|
const def = alarm('browser.tabs');
|
|
489
|
+
// Said either way, never left out: with no line configured (the default) the count was read and is past no line,
|
|
490
|
+
// a positive observation that clears an alarm raised under a line since removed.
|
|
431
491
|
if (def.warn != null) slow.observations.push({ key: 'browser.tabs', family: 'tabs', def, value: tabs.length, unit: 'tabs', summary: `${tabs.length} browser tabs open` });
|
|
492
|
+
else slow.observations.push({ key: 'browser.tabs', family: 'tabs', def, value: tabs.length, unit: 'tabs', met: { warn: false, crit: false, clears: true }, summary: `${tabs.length} browser tabs open (no line configured)` });
|
|
432
493
|
});
|
|
433
494
|
|
|
434
495
|
if (on.has('power')) await attempt('power', async () => {
|
|
@@ -437,7 +498,8 @@ export class Sampler {
|
|
|
437
498
|
const level = /pressure level:\s*(\w+)/i.exec(r.stdout)?.[1] ?? null;
|
|
438
499
|
slow.readings.power = { thermalPressure: level };
|
|
439
500
|
slow.families.add('power');
|
|
440
|
-
if (level) slow.observations.push({
|
|
501
|
+
if (!level) slow.observations.push({ unreadKey: 'power.thermal', family: 'power', why: 'the thermal pressure was not read this cycle' });
|
|
502
|
+
else slow.observations.push({ key: 'power.thermal', family: 'power', def: alarm('power.thermal'), value: level, met: { warn: level !== 'Nominal', crit: /Heavy|Trapping|Sleeping/.test(level), clears: level === 'Nominal' }, summary: `thermal pressure ${level}` });
|
|
441
503
|
});
|
|
442
504
|
}
|
|
443
505
|
|
package/lib/thresholds.mjs
CHANGED
|
@@ -5,6 +5,7 @@ export const DEFAULTS = {
|
|
|
5
5
|
slowIntervalSec: 60, // disk space, services, the connector, tabs
|
|
6
6
|
footprintTopEverySec: 300, // macOS: every user's largest footprints through top (about 2 s of top per run)
|
|
7
7
|
repeatMin: 60, // a raised alarm mails again this often (netdata's repeat cap)
|
|
8
|
+
unknownMin: 15, // a reading unknown this long (raised or not) is mailed to the maintainers, and again each repeatMin
|
|
8
9
|
historyHours: 24,
|
|
9
10
|
daemons: {
|
|
10
11
|
darwin: ['dasd', 'fseventsd', 'mds', 'mds_stores', 'mdworker_shared', 'WindowServer', 'kernel_task', 'backupd', 'softwareupdated', 'photoanalysisd', 'cloudd', 'bird', 'nsurlsessiond', 'trustd', 'syspolicyd', 'XprotectService', 'logd', 'runningboardd', 'launchservicesd', 'coreduetd', 'spotlightknowledged'],
|
|
@@ -48,7 +49,8 @@ export function merge(base, over) {
|
|
|
48
49
|
|
|
49
50
|
/** Which levels a numeric value meets under a threshold definition. */
|
|
50
51
|
export function meets(value, def) {
|
|
51
|
-
|
|
52
|
+
// no value is no reading: unknown, which neither raises nor clears (D190 class 1)
|
|
53
|
+
if (value == null || Number.isNaN(value)) return { warn: false, crit: false, clears: false, unknown: true };
|
|
52
54
|
const past = (line) => line != null && (def.below ? value < line : value > line);
|
|
53
55
|
const crit = past(def.crit);
|
|
54
56
|
const warn = crit || past(def.warn);
|
|
@@ -59,16 +61,56 @@ export function meets(value, def) {
|
|
|
59
61
|
const RANK = { none: 0, warn: 1, crit: 2 };
|
|
60
62
|
|
|
61
63
|
/**
|
|
62
|
-
* One step of every alarm.
|
|
63
|
-
*
|
|
64
|
-
*
|
|
64
|
+
* One step of every alarm. A raised alarm clears only on a positive observation (D190 class 1, held structurally:
|
|
65
|
+
* no read site has to say it failed for its alarm to hold):
|
|
66
|
+
* - its value, read this cycle, is past its clear line;
|
|
67
|
+
* - the read that judges it covered it with a real value and found it under its line (`{ covered: prefix, family,
|
|
68
|
+
* entities | all }`: every key `prefix + entity` was evaluated, so one not observed is under its line);
|
|
69
|
+
* - a complete enumeration of its family no longer has its entity (`{ enumerated: family, by: 'pid' | 'suffix',
|
|
70
|
+
* entities }`: the process list, the mounts, the configured services);
|
|
71
|
+
* - its feature is off (`{ off: family }`).
|
|
72
|
+
* Anything else not observed is unknown: the alarm holds, saying since when and why (`{ unreadFamily | unreadKey |
|
|
73
|
+
* unreadPrefix, why }` give the why where a site knows it; otherwise it is "not observed this cycle").
|
|
74
|
+
* `observations` also carry the readings: [{ key, family, def, value, met?, unit, summary, detail }].
|
|
75
|
+
* Every unknown (raised or not: a key or a family that could not be read) is kept in `unknowns` (key → { why,
|
|
76
|
+
* since }) for its notification (probe.mjs); one read again leaves it.
|
|
77
|
+
* Returns the transitions; `state` (key → record) and `unknowns` are updated in place.
|
|
65
78
|
*/
|
|
66
|
-
export function judge(state, observations, families, now) {
|
|
79
|
+
export function judge(state, observations, families, now, unknowns = {}) {
|
|
67
80
|
const transitions = [];
|
|
68
81
|
const seen = new Set();
|
|
82
|
+
const unreadFamilies = new Map(), unreadKeys = new Map(), unreadPrefixes = [], covered = [], enumerated = new Map(), off = new Set();
|
|
69
83
|
for (const o of observations) {
|
|
84
|
+
if (o.unreadFamily) unreadFamilies.set(o.unreadFamily, o.why);
|
|
85
|
+
else if (o.unreadKey) unreadKeys.set(o.unreadKey, { family: o.family, why: o.why });
|
|
86
|
+
else if (o.unreadPrefix) unreadPrefixes.push(o);
|
|
87
|
+
else if (o.covered) covered.push(o);
|
|
88
|
+
else if (o.enumerated) enumerated.set(o.enumerated, o);
|
|
89
|
+
else if (o.off) off.add(o.off);
|
|
90
|
+
}
|
|
91
|
+
const why = (key, family) => unreadFamilies.get(family) ?? unreadKeys.get(key)?.why ?? unreadPrefixes.find((p) => p.family === family && key.startsWith(p.unreadPrefix))?.why ?? null;
|
|
92
|
+
// an entity is a pid (a number) or a name: a key's suffix matches either form
|
|
93
|
+
const holds = (set, entity) => set.has(entity) || (/^\d+$/.test(entity) && set.has(Number(entity)));
|
|
94
|
+
const under = (key, family) => covered.some((c) => c.family === family && key.startsWith(c.covered) && (c.all || (c.entities && holds(c.entities, key.slice(c.covered.length)))));
|
|
95
|
+
const gone = (key, family) => {
|
|
96
|
+
const e = enumerated.get(family); if (!e) return false;
|
|
97
|
+
const entity = e.by === 'pid' ? Number(key.slice(key.lastIndexOf(':') + 1)) : key.slice(key.indexOf(':') + 1);
|
|
98
|
+
return !e.entities.has(entity);
|
|
99
|
+
};
|
|
100
|
+
const hold = (key, reason) => {
|
|
101
|
+
const s = state[key];
|
|
102
|
+
if (s) Object.assign(s, { unread: String(reason), unreadAt: s.unreadAt ?? now });
|
|
103
|
+
unknowns[key] = { family: s?.family ?? null, why: String(reason), since: unknowns[key]?.since ?? now };
|
|
104
|
+
};
|
|
105
|
+
// what a site said it could not read is unknown whether or not it was ever raised (its notification: probe.mjs)
|
|
106
|
+
const read = new Set();
|
|
107
|
+
for (const [family, reason] of unreadFamilies) { const key = `family:${family}`; unknowns[key] = { family, why: reason, since: unknowns[key]?.since ?? now }; read.add(key); }
|
|
108
|
+
for (const [key, { family, why: reason }] of unreadKeys) { unknowns[key] = { family, why: reason, since: unknowns[key]?.since ?? now }; read.add(key); }
|
|
109
|
+
for (const o of observations) {
|
|
110
|
+
if (!o.key) continue;
|
|
70
111
|
seen.add(o.key);
|
|
71
112
|
const m = o.met ?? meets(o.value, o.def);
|
|
113
|
+
if (m.unknown) { hold(o.key, `${o.key}: no value read this cycle`); read.add(o.key); continue; }
|
|
72
114
|
const s = state[o.key] ?? (state[o.key] = { key: o.key, family: o.family, level: 'none', warnSince: null, critSince: null, raisedAt: null, lastMailAt: null });
|
|
73
115
|
s.warnSince = m.warn ? (s.warnSince ?? now) : null;
|
|
74
116
|
s.critSince = m.crit ? (s.critSince ?? now) : null;
|
|
@@ -79,7 +121,7 @@ export function judge(state, observations, families, now) {
|
|
|
79
121
|
else if (m.warn && now - s.warnSince >= warnFor) target = 'warn';
|
|
80
122
|
// Hysteresis: a raised alarm holds at warn until the value is past its clear line.
|
|
81
123
|
if (target === 'none' && s.level !== 'none' && !m.clears) target = 'warn';
|
|
82
|
-
Object.assign(s, { value: o.value, unit: o.unit ?? null, summary: o.summary, detail: o.detail ?? null, at: now });
|
|
124
|
+
Object.assign(s, { value: o.value, unit: o.unit ?? null, summary: o.summary, detail: o.detail ?? null, at: now, unread: null, unreadAt: null });
|
|
83
125
|
if (target !== s.level) {
|
|
84
126
|
transitions.push({ key: o.key, from: s.level, to: target, value: o.value, unit: o.unit ?? null, summary: o.summary, detail: o.detail ?? null });
|
|
85
127
|
if (RANK[target] > RANK[s.level] && s.level === 'none') s.raisedAt = now;
|
|
@@ -88,16 +130,27 @@ export function judge(state, observations, families, now) {
|
|
|
88
130
|
}
|
|
89
131
|
}
|
|
90
132
|
for (const [key, s] of Object.entries(state)) {
|
|
91
|
-
if (seen.has(key)
|
|
92
|
-
//
|
|
93
|
-
|
|
94
|
-
if (
|
|
133
|
+
if (seen.has(key)) continue;
|
|
134
|
+
// a key not observed clears only on a positive observation; anything else is unknown
|
|
135
|
+
const cleared = off.has(s.family) ? 'its feature is off' : under(key, s.family) ? 'under its line' : gone(key, s.family) ? 'it is gone' : null;
|
|
136
|
+
if (!cleared) {
|
|
137
|
+
if (s.level === 'none') { delete state[key]; continue; } // nothing raised to hold
|
|
138
|
+
// An alarm restored from before a restart waits out the first samples, which have no rates yet.
|
|
139
|
+
if (s.restoredUntil && now < s.restoredUntil) continue;
|
|
140
|
+
hold(key, why(key, s.family) ?? `not observed this cycle: no complete read of its ${s.family} family covered it`);
|
|
141
|
+
read.add(key);
|
|
142
|
+
continue;
|
|
143
|
+
}
|
|
144
|
+
if (s.level !== 'none') transitions.push({ key, from: s.level, to: 'none', value: null, summary: `${s.summary} — cleared: ${cleared}`, detail: s.detail, gone: cleared === 'it is gone' });
|
|
95
145
|
delete state[key];
|
|
96
146
|
}
|
|
147
|
+
// an unknown read again (or cleared) this cycle leaves the list
|
|
148
|
+
for (const key of Object.keys(unknowns)) if (!read.has(key)) delete unknowns[key];
|
|
97
149
|
return transitions;
|
|
98
150
|
}
|
|
99
151
|
|
|
100
152
|
export function raised(state) {
|
|
101
153
|
return Object.values(state).filter((s) => s.level !== 'none').sort((a, b) => RANK[b.level] - RANK[a.level] || a.key.localeCompare(b.key))
|
|
102
|
-
.map(({ key, family, level, value, unit, summary, detail, raisedAt }) => ({ key, family, level, value, unit, summary, detail, since: raisedAt ? new Date(raisedAt).toISOString() : null
|
|
154
|
+
.map(({ key, family, level, value, unit, summary, detail, raisedAt, unread, unreadAt }) => ({ key, family, level, value, unit, summary, detail, since: raisedAt ? new Date(raisedAt).toISOString() : null,
|
|
155
|
+
...(unread ? { unknown: { why: unread, since: new Date(unreadAt).toISOString() } } : {}) }));
|
|
103
156
|
}
|
package/lib/win32.mjs
CHANGED
|
@@ -5,7 +5,9 @@ import { run } from './run.mjs';
|
|
|
5
5
|
|
|
6
6
|
export async function memory() {
|
|
7
7
|
const total = totalmem();
|
|
8
|
-
|
|
8
|
+
// what Node gives no reading of on Windows: left out of judging, never unknown
|
|
9
|
+
return { pressureLevel: null, totalBytes: total, freePct: Math.round((freemem() / total) * 100), swapUsedBytes: null, swapTotalBytes: null, counters: {}, pageSize: 4096,
|
|
10
|
+
absent: ['memory.pressure', 'memory.swapping', 'memory.swapped30m', 'memory.compressions'] };
|
|
9
11
|
}
|
|
10
12
|
|
|
11
13
|
function splitAddress(text) {
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@volter/supercode-health",
|
|
3
|
-
"version": "0.1.
|
|
3
|
+
"version": "0.1.5",
|
|
4
4
|
"type": "module",
|
|
5
5
|
"description": "Optional machine pack: a cheap, deterministic probe of a machine's vitals that mails the machine's maintainer when a threshold is crossed. Each feature sits behind its own permission barrier.",
|
|
6
6
|
"license": "MIT",
|