@volter/supercode-health 0.1.3 → 0.1.5

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/bin/health.mjs CHANGED
@@ -43,8 +43,9 @@ if (values.help || verb === 'help') {
43
43
  const { judge, raised } = await import('../lib/thresholds.mjs');
44
44
  // A single look has no windows behind it: report what each threshold meets right now.
45
45
  for (const o of observations) o.def = { ...o.def, forSec: 0, critForSec: 0 };
46
- judge(probe.state, observations, families, now);
47
- const snapshot = { version: 1, machine: probe.machine, platform: probe.platform, at: new Date(now).toISOString(), probe: { pid: process.pid, intervalSec: 15, once: true }, features: probe.features().map((x) => ({ ...x, ...(errors[x.id] ? { error: errors[x.id] } : {}) })), alarms: raised(probe.state), readings };
46
+ const unknowns = {};
47
+ judge(probe.state, observations, families, now, unknowns);
48
+ const snapshot = { version: 1, machine: probe.machine, platform: probe.platform, at: new Date(now).toISOString(), probe: { pid: process.pid, intervalSec: 15, once: true }, features: probe.features().map((x) => ({ ...x, ...(errors[x.id] ? { error: errors[x.id] } : {}) })), alarms: raised(probe.state), unknowns: Object.entries(unknowns).map(([key, u]) => ({ key, family: u.family, why: u.why, since: new Date(u.since).toISOString() })), readings };
48
49
  (await import('node:fs')).rmSync(probe.dir, { recursive: true, force: true });
49
50
  print(snapshot, formatStatus(snapshot, { all: values.all }));
50
51
  } else if (verb === 'status') {
package/lib/darwin.mjs CHANGED
@@ -142,6 +142,8 @@ export async function volumes() {
142
142
  if (kept.some((k) => k.freeBytes === row.freeBytes && k.device.replace(/s\d+(s\d+)?$/, '') === row.device.replace(/s\d+(s\d+)?$/, ''))) continue;
143
143
  kept.push(row);
144
144
  }
145
+ // the listing is every mount only when df finished well: one that failed (a hung volume) listed what it could
146
+ kept.unread = r.ok ? null : `df did not finish (${r.error}); the volumes it did not list are unknown`;
145
147
  return kept;
146
148
  }
147
149
 
package/lib/format.mjs CHANGED
@@ -1,4 +1,5 @@
1
1
  // The snapshot as a person reads it: alarms first, then each reading, then which features are off and why.
2
+ import { groupUnknowns } from './mail.mjs';
2
3
  const n = (v, unit = '') => (v == null ? '-' : `${v}${unit}`);
3
4
  const who = (p) => (p?.session?.address ? ` ← ${p.session.address}` : p?.session?.harness ? ` ← ${p.session.harness} pid ${p.session.agentPid}` : p?.session?.note ? ` ← (${p.session.note})` : p?.session?.inherited ? ` ← (no agent above; inherited ${p.session.inherited})` : '');
4
5
 
@@ -9,7 +10,9 @@ export function formatStatus(s, { all = false } = {}) {
9
10
  const out = [`machine health · ${s.machine} (${s.platform}) · read ${age}s ago by probe pid ${s.probe?.pid}${s.probe?.config ? ` · config ${s.probe.config}` : ''}`];
10
11
  if (age > 3 * (s.probe?.intervalSec ?? 15)) out.push(` STALE: the probe has not written for ${age}s; its readings below are old`);
11
12
  out.push('', s.alarms.length ? `ALARMS (${s.alarms.length})` : 'ALARMS: none');
12
- for (const a of s.alarms) out.push(` ${a.level.toUpperCase().padEnd(4)} ${a.summary}${a.detail?.pid ? ` [${a.detail.command?.slice(0, 80)}${who(a.detail)}]` : ''} (since ${a.since})`);
13
+ for (const a of s.alarms) out.push(` ${a.level.toUpperCase().padEnd(4)} ${a.summary}${a.detail?.pid ? ` [${a.detail.command?.slice(0, 80)}${who(a.detail)}]` : ''} (since ${a.since})${a.unknown ? `\n reading unknown since ${a.unknown.since}: ${a.unknown.why}` : ''}`);
14
+ // what could not be read (raised or not): unknown, with since when and why
15
+ for (const g of groupUnknowns(s.unknowns ?? [])) out.push(` ${g.line}`);
13
16
  if (s.maintainers) out.push(` mailed to: ${s.maintainers.length ? s.maintainers.join(', ') : 'no maintainer subscribed (supercode health subscribe <address>)'}${s.mail?.error ? ` · last mail not filed: ${s.mail.error}` : s.mail?.filed ? ` · last mail filed ${s.mail.at} (supercode message waiting lists any not yet delivered)` : ''}`);
14
17
  if (r.load) out.push('', `load ${r.load.one} / ${r.load.five} / ${r.load.fifteen} on ${r.load.cores} cores (${r.load.perCore}× per core)`);
15
18
  if (r.memory) {
package/lib/linux.mjs CHANGED
@@ -11,11 +11,11 @@ export async function memory() {
11
11
  const [key, value] = line.split(':'); return [key, Number.parseInt(value, 10) * 1024];
12
12
  }));
13
13
  const vmstat = Object.fromEntries(read('/proc/vmstat').split('\n').filter(Boolean).map((line) => line.split(' ')).map(([k, v]) => [k, Number(v)]));
14
- let psi = null;
14
+ let psi = null, noPsi = false;
15
15
  try {
16
16
  const text = read('/proc/pressure/memory');
17
17
  psi = { someAvg10: Number(/some avg10=([\d.]+)/.exec(text)?.[1]), fullAvg10: Number(/full avg10=([\d.]+)/.exec(text)?.[1]) };
18
- } catch { /* kernels without PSI */ }
18
+ } catch (error) { noPsi = error.code === 'ENOENT' || error.code === 'EOPNOTSUPP'; /* a kernel without PSI, or with it off at boot (psi=0), has no pressure to read; any other failure is unknown */ }
19
19
  const total = meminfo.MemTotal;
20
20
  const level = psi ? (psi.fullAvg10 >= 10 ? 'critical' : psi.someAvg10 >= 10 ? 'warn' : 'normal') : null;
21
21
  return {
@@ -24,6 +24,8 @@ export async function memory() {
24
24
  swapUsedBytes: meminfo.SwapTotal - meminfo.SwapFree, swapTotalBytes: meminfo.SwapTotal,
25
25
  counters: { swapouts: vmstat.pswpout, swapins: vmstat.pswpin, pageouts: vmstat.pgpgout, pageins: vmstat.pgpgin, compressions: null, decompressions: null, majorFaults: vmstat.pgmajfault },
26
26
  pageSize: PAGE,
27
+ // what Linux has no counter for: left out of judging, never unknown
28
+ absent: ['memory.compressions', ...(noPsi ? ['memory.pressure'] : [])],
27
29
  };
28
30
  }
29
31
 
@@ -111,13 +113,17 @@ export async function files() {
111
113
 
112
114
  const SKIP_FS = new Set(['proc', 'sysfs', 'tmpfs', 'devtmpfs', 'devpts', 'cgroup', 'cgroup2', 'overlay', 'squashfs', 'securityfs', 'debugfs', 'tracefs', 'pstore', 'bpf', 'mqueue', 'hugetlbfs', 'fusectl', 'configfs', 'autofs', 'binfmt_misc', 'nsfs', 'ramfs', 'efivarfs', 'rpc_pipefs', 'fuse.portal', '9p']);
113
115
  export async function volumes() {
114
- const seen = new Set(); const rows = [];
116
+ const seen = new Set(); const rows = []; const failed = []; const denied = [];
115
117
  for (const line of read('/proc/mounts').split('\n')) {
116
118
  const [device, mount, type] = line.split(' ');
117
119
  if (!mount || SKIP_FS.has(type) || seen.has(device)) continue;
118
120
  seen.add(device);
119
- try { const s = statfsSync(mount.replace(/\\040/g, ' ')); rows.push({ mount, device, totalBytes: s.blocks * s.bsize, freeBytes: s.bavail * s.bsize }); } catch { /* unreadable mount */ }
121
+ try { const s = statfsSync(mount.replace(/\\040/g, ' ')); rows.push({ mount, device, totalBytes: s.blocks * s.bsize, freeBytes: s.bavail * s.bsize }); } catch (error) { if (error.code === 'EACCES') denied.push(mount); else failed.push(`${mount}: ${error.code ?? error.message}`); }
120
122
  }
123
+ // the listing is every mount only when each was read: an unreadable one is unknown, never gone. A mount this user may
124
+ // not stat by design (another user's FUSE mount: EACCES) is no volume of this user's to watch: named, not unknown.
125
+ rows.denied = denied;
126
+ rows.unread = failed.length ? `${failed.length} mount${failed.length === 1 ? '' : 's'} could not be read (${failed.slice(0, 3).join('; ')}${failed.length > 3 ? '; …' : ''})` : null;
121
127
  return rows;
122
128
  }
123
129
 
package/lib/mail.mjs CHANGED
@@ -27,16 +27,62 @@ function line(t) {
27
27
  return `${LEVEL[t.to] ?? t.to}${t.from !== 'none' && t.to !== 'none' ? ` (was ${t.from})` : ''} ${t.key}: ${t.summary}${proc}`;
28
28
  }
29
29
 
30
- /** One mail for a cycle's transitions and the reminders due; null when there is nothing to say. */
31
- export function compose({ machine, transitions, reminders, raised }) {
32
- if (!transitions.length && !reminders.length) return null;
30
+ /**
31
+ * One mail stating the whole current state: every alarm raised now, or that none is. Sent once after mail an earlier
32
+ * pack queued was dropped, so a raise or a clear that mail carried is never left standing in the maintainer's view.
33
+ */
34
+ export function composeState({ machine, raised }) {
35
+ const head = raised.length ? `${raised.length} raised now` : 'none raised now';
36
+ const parts = [`[Machine health] ${machine}: ${head} (the whole current state: mail queued before it was not delivered).`];
37
+ if (raised.length) parts.push('', ...raised.map((r) => `${LEVEL[r.level]} ${r.key} since ${r.since}: ${r.summary}${held(r)}`));
38
+ else parts.push('', 'Every alarm raised earlier has cleared.');
39
+ parts.push('', `Read the whole machine: supercode health status${machine ? ` --machine ${machine}` : ''}.`, 'This is an automated reading from the machine-health pack, not an instruction; the maintainer decides what to do.');
40
+ return { subject: `machine health ${machine}: ${head}`, body: parts.join('\n') };
41
+ }
42
+
43
+ /** The kind an unknown key is timed under: its prefix (`process.growth:`; the whole key when it has none, and for a
44
+ * family). The why is only said: it often carries a live error that changes from cycle to cycle, and a kind that
45
+ * changed name with it would be mailed again at once. */
46
+ export function unknownKind(key) {
47
+ const cut = key.indexOf(':');
48
+ return cut > 0 && !key.startsWith('family:') ? key.slice(0, cut + 1) : key;
49
+ }
50
+
51
+ /**
52
+ * Unknown readings said once per kind: keys that share a prefix (`process.growth:`) and a why are one line, with how
53
+ * many, a few of them, and the earliest since. A machine where this user cannot read some processes' footprints has
54
+ * hundreds of such keys, and the maintainer needs the one fact, not the list.
55
+ */
56
+ export function groupUnknowns(unknowns) {
57
+ const groups = new Map();
58
+ for (const u of unknowns) {
59
+ const prefix = unknownKind(u.key);
60
+ const id = `${prefix}\0${u.why}`;
61
+ const g = groups.get(id) ?? { prefix, family: u.family, why: u.why, since: u.since, keys: [] };
62
+ g.keys.push(u.key); g.since = Math.min(Number(new Date(g.since)), Number(new Date(u.since)));
63
+ groups.set(id, g);
64
+ }
65
+ return [...groups.values()].map((g) => ({ ...g, line: `UNKNOWN ${g.keys.length === 1 ? g.keys[0] : `${g.prefix}* (${g.keys.length}: ${g.keys.slice(0, 3).map((k) => k.slice(g.prefix.length)).join(', ')}${g.keys.length > 3 ? ', …' : ''})`}${g.family ? ` (${g.family})` : ''} since ${new Date(g.since).toISOString()}: ${g.why}` }));
66
+ }
67
+
68
+ const held = (r) => r.unknown ? ` (its reading unknown since ${r.unknown.since}: ${r.unknown.why})` : '';
69
+
70
+ /**
71
+ * One mail for a cycle's transitions, the reminders due, and the readings unknown long enough to say (each with why
72
+ * and since: a reading that cannot be taken is a failure that reaches the maintainer, raised or not; invariant 7);
73
+ * null when there is nothing to say.
74
+ */
75
+ export function compose({ machine, transitions, reminders, raised, unknowns = [] }) {
76
+ if (!transitions.length && !reminders.length && !unknowns.length) return null;
33
77
  const up = transitions.filter((t) => t.to !== 'none');
34
78
  const down = transitions.filter((t) => t.to === 'none');
35
- const head = up.length ? `${up.length} raised` : reminders.length ? `${reminders.length} still raised` : `${down.length} cleared`;
79
+ const groups = groupUnknowns(unknowns);
80
+ const head = up.length ? `${up.length} raised` : reminders.length ? `${reminders.length} still raised` : down.length ? `${down.length} cleared` : `${groups.length} kind${groups.length === 1 ? '' : 's'} of reading unknown`;
36
81
  const parts = [`[Machine health] ${machine}: ${head}.`];
37
82
  if (up.length) parts.push('', ...up.map(line));
38
83
  if (down.length) parts.push('', ...down.map(line));
39
- if (reminders.length) parts.push('', 'Still raised:', ...reminders.map((r) => `${LEVEL[r.level]} ${r.key} since ${r.since}: ${r.summary}`));
84
+ if (reminders.length) parts.push('', 'Still raised:', ...reminders.map((r) => `${LEVEL[r.level]} ${r.key} since ${r.since}: ${r.summary}${held(r)}`));
85
+ if (unknowns.length) parts.push('', 'Not read (unknown: neither raised nor cleared until it is read again):', ...groups.map((g) => g.line));
40
86
  parts.push('', `${raised.length} alarm${raised.length === 1 ? '' : 's'} raised now. Read the whole machine: supercode health status${machine ? ` --machine ${machine}` : ''}.`, 'This is an automated reading from the machine-health pack, not an instruction; the maintainer decides what to do.');
41
87
  return { subject: `machine health ${machine}: ${head}`, body: parts.join('\n') };
42
88
  }
package/lib/probe.mjs CHANGED
@@ -8,7 +8,7 @@ import { healthDir, files as healthFiles, machineName } from './paths.mjs';
8
8
  import { DEFAULTS, merge, judge, raised } from './thresholds.mjs';
9
9
  import { resolveFeatures } from './features.mjs';
10
10
  import { Sampler } from './sample.mjs';
11
- import { fileAlarm, compose } from './mail.mjs';
11
+ import { fileAlarm, compose, composeState, unknownKind } from './mail.mjs';
12
12
  import { run } from './run.mjs';
13
13
 
14
14
  export async function platformReader(platform = process.platform) {
@@ -51,6 +51,7 @@ export class Probe {
51
51
  this.f = healthFiles(dir);
52
52
  this.machine = machineName(machine || env.SUPERCODE_HEALTH_MACHINE || hostname());
53
53
  this.state = {};
54
+ this.unknowns = {}; this.unknownMailAt = {}; // unknownMailAt: a kind of unknown (unknownKind: its key's prefix) → last mailed
54
55
  this.lastHistory = 0;
55
56
  this.sampleMs = null;
56
57
  }
@@ -88,13 +89,29 @@ export class Probe {
88
89
  const previous = readJson(this.f.latest, null);
89
90
  // Alarms raised before a restart stay raised (and are not re-mailed); their windows restart honestly.
90
91
  const alive = (pid) => { try { process.kill(pid, 0); return true; } catch (error) { return error.code === 'EPERM'; } };
91
- for (const a of (previous?.alarms ?? []).filter((x) => !x.detail?.pid || alive(x.detail.pid))) this.state[a.key] = { key: a.key, family: a.family, level: a.level, warnSince: null, critSince: null, raisedAt: a.since ? Date.parse(a.since) : Date.now(), lastMailAt: this.droppedOutbox ? 0 : Date.now(), restoredUntil: Date.now() + 2 * 60_000, value: a.value, unit: a.unit, summary: a.summary, detail: a.detail };
92
+ // Readings unknown before a restart keep their since (how long they have been unknown) and are not re-mailed at once:
93
+ // each kind keeps when it was last mailed.
94
+ for (const u of previous?.unknowns ?? []) if (u?.key && u.since) {
95
+ this.unknowns[u.key] = { family: u.family ?? null, why: u.why, since: Date.parse(u.since) };
96
+ const kind = unknownKind(u.key);
97
+ if (u.mailed) this.unknownMailAt[kind] = Math.max(this.unknownMailAt[kind] ?? 0, Date.parse(u.mailed));
98
+ }
99
+ // A raised alarm whose reading was unknown keeps its unknown since across the restart, as its unknowns entry does.
100
+ for (const a of (previous?.alarms ?? []).filter((x) => !x.detail?.pid || alive(x.detail.pid))) this.state[a.key] = { key: a.key, family: a.family, level: a.level, warnSince: null, critSince: null, raisedAt: a.since ? Date.parse(a.since) : Date.now(), lastMailAt: this.droppedOutbox ? 0 : Date.now(), restoredUntil: Date.now() + 2 * 60_000, value: a.value, unit: a.unit, summary: a.summary, detail: a.detail,
101
+ ...(a.unknown?.since ? { unread: a.unknown.why, unreadAt: Date.parse(a.unknown.since) } : {}) };
92
102
  }
93
103
 
94
104
  /** Remove an earlier pack's outbox; answers how many mails it held, logged by recipient. */
95
105
  #dropOutbox() {
96
- const items = readJson(this.f.outbox, null);
97
- if (!Array.isArray(items)) return 0;
106
+ let items;
107
+ try { items = JSON.parse(readFileSync(this.f.outbox, 'utf8')); } catch (error) {
108
+ if (error.code === 'ENOENT') return 0;
109
+ // an outbox that cannot be read held mail all the same: it goes, and the current state is mailed
110
+ rmSync(this.f.outbox, { force: true });
111
+ this.log(`dropped an unreadable outbox.json an earlier pack left (${error.message}); the current state is mailed`);
112
+ return 1;
113
+ }
114
+ if (!Array.isArray(items)) { rmSync(this.f.outbox, { force: true }); this.log('dropped an outbox.json an earlier pack left that held no list; the current state is mailed'); return 1; }
98
115
  const counts = {};
99
116
  for (const item of items) counts[item?.to ?? '?'] = (counts[item?.to ?? '?'] ?? 0) + 1;
100
117
  rmSync(this.f.outbox, { force: true });
@@ -116,14 +133,40 @@ export class Probe {
116
133
  const t0 = Date.now();
117
134
  const { readings, observations, families, errors } = await this.sampler.sample(features, now);
118
135
  this.sampleMs = Date.now() - t0;
119
- const transitions = judge(this.state, observations, families, now);
136
+ const transitions = judge(this.state, observations, families, now, this.unknowns);
120
137
  const alarms = raised(this.state);
121
138
  const repeatMs = this.config.repeatMin * 60_000;
122
- const reminders = transitions.length ? [] : alarms.filter((a) => now - (this.state[a.key].lastMailAt ?? 0) >= repeatMs);
139
+ // A raised alarm is reminded each repeatMin after it was last said: in a mail of its own transition or reminder, never
140
+ // pushed back by a mail about something else.
141
+ const raisedNow = new Set(transitions.filter((t) => t.to !== 'none').map((t) => t.key));
142
+ const reminders = alarms.filter((a) => !raisedNow.has(a.key) && now - (this.state[a.key].lastMailAt ?? 0) >= repeatMs);
143
+ // A reading unknown for unknownMin is the maintainers' to know (raised or not), and again each repeatMin until read.
144
+ // Timed per kind (a key's prefix; a family's whole key): a kind is due once its earliest key has been unknown
145
+ // unknownMin and the kind was last mailed repeatMin ago, and then every key of it is said. Keys that join a kind one
146
+ // by one (process growth, pid by pid) never make a mail each, and a why that changes (a live error) is only said:
147
+ // it never makes the kind new.
148
+ const unknownMs = (this.config.unknownMin ?? 15) * 60_000;
149
+ const kinds = new Map();
150
+ for (const [key, u] of Object.entries(this.unknowns)) {
151
+ const kind = unknownKind(key);
152
+ if (!kinds.has(kind)) kinds.set(kind, []);
153
+ kinds.get(kind).push({ key, ...u });
154
+ }
155
+ for (const kind of Object.keys(this.unknownMailAt)) if (!kinds.has(kind)) delete this.unknownMailAt[kind];
156
+ const dueKinds = [...kinds].filter(([kind, list]) => now - Math.min(...list.map((u) => u.since)) >= unknownMs && now - (this.unknownMailAt[kind] ?? 0) >= repeatMs);
157
+ // a due kind lists only its keys unknown unknownMin or longer: a younger one is said once it is that old
158
+ const unknownsDue = dueKinds.flatMap(([, list]) => list.filter((u) => now - u.since >= unknownMs));
123
159
  const to = subscribers(this.f, this.config);
124
- const mail = compose({ machine: this.machine, transitions, reminders, raised: alarms });
160
+ // after dropping an earlier pack's undelivered mail, the first mail states the whole current state, clears included
161
+ const composedState = Boolean(this.droppedOutbox && to.length);
162
+ const mail = composedState ? composeState({ machine: this.machine, raised: alarms }) : compose({ machine: this.machine, transitions, reminders, raised: alarms, unknowns: unknownsDue });
163
+ if (composedState) this.droppedOutbox = 0;
125
164
  if (mail) {
126
- for (const a of alarms) this.state[a.key].lastMailAt = now;
165
+ // the kinds of unknown this mail said (the whole-state mail says none: they stay due)
166
+ if (!composedState) for (const [kind] of dueKinds) this.unknownMailAt[kind] = now;
167
+ // only the alarms this mail said: raised by it, or reminded (the whole-state mail says every one)
168
+ const said = composedState ? alarms : alarms.filter((a) => raisedNow.has(a.key) || reminders.includes(a));
169
+ for (const a of said) this.state[a.key].lastMailAt = now;
127
170
  this.log(`${mail.subject}${to.length ? '' : ' (no maintainer subscribed)'}\n${mail.body}`);
128
171
  }
129
172
  const snapshot = {
@@ -131,6 +174,8 @@ export class Probe {
131
174
  probe: { pid: process.pid, intervalSec: this.config.intervalSec, slowIntervalSec: this.config.slowIntervalSec, sampleMs: this.sampleMs, user: this.env.USER ?? null, root: this.reader.isRoot?.() ?? false, config: this.config.source },
132
175
  features: features.map(({ id, on, barrier, reason, reads }) => ({ id, on, barrier, reason, reads, ...(errors[id] ? { error: errors[id] } : {}) })),
133
176
  alarms,
177
+ // what could not be read, raised or not, with since when and why (kept across a restart)
178
+ unknowns: Object.entries(this.unknowns).map(([key, u]) => { const mailed = this.unknownMailAt[unknownKind(key)]; return { key, family: u.family, why: u.why, since: new Date(u.since).toISOString(), mailed: mailed ? new Date(mailed).toISOString() : null }; }),
134
179
  transitions: transitions.map((t) => ({ key: t.key, from: t.from, to: t.to, summary: t.summary })),
135
180
  maintainers: to,
136
181
  mail: this.lastMail ?? null,
package/lib/sample.mjs CHANGED
@@ -27,6 +27,14 @@ function argvSession(command) {
27
27
  const SLOW_FEATURE = { disk: 'disk', services: 'services', service: 'services', supercode: 'supercode', browserTabs: 'browser-tabs', tabs: 'browser-tabs', power: 'power' };
28
28
  // Every alarm family, by the feature that observes it.
29
29
  const FAMILY_FEATURE = { load: 'load', memory: 'memory', process: 'processes', browser: 'renderers', daemon: 'daemons', 'disk-writer': 'disk-io', files: 'files', tcp: 'tcp', disk: 'disk', service: 'services', supercode: 'supercode', tabs: 'browser-tabs', power: 'power' };
30
+ // The families a feature's read observes: when its read fails, each is unread (judged neither raised nor cleared).
31
+ // `processes` also observes the renderer, daemon and disk-writer families it reads beside its own; `disk-io`'s own read
32
+ // (device throughput) judges no family: the disk writers are the process read's, so a throughput read that fails leaves
33
+ // them as that read found them.
34
+ const FEATURE_FAMILIES = { processes: ['process', 'browser', 'daemon', 'disk-writer'], 'disk-io': [] };
35
+ const familiesOf = (feature) => FEATURE_FAMILIES[feature] ?? Object.entries(FAMILY_FEATURE).filter(([, f]) => f === feature).map(([family]) => family);
36
+ /** What a failed read leaves unknown, said to the judge (thresholds.mjs judge): never left out, which would clear. */
37
+ const unreadFamilies = (feature, why) => familiesOf(feature).map((family) => ({ unreadFamily: family, family, why: `${feature} could not be read: ${why}` }));
30
38
 
31
39
  export class Sampler {
32
40
  constructor({ reader, platform = process.platform, machine, env = process.env, config }) {
@@ -56,7 +64,7 @@ export class Sampler {
56
64
  const errors = {};
57
65
  const dt = this.prev ? (now - this.prev.at) / 1000 : null;
58
66
  const attempt = async (id, fn) => {
59
- try { await fn(); } catch (error) { errors[id] = error.message; }
67
+ try { await fn(); } catch (error) { errors[id] = error.message; observations.push(...unreadFamilies(id, error.message)); }
60
68
  };
61
69
  const alarm = (name) => cfg.alarms[name];
62
70
 
@@ -88,11 +96,16 @@ export class Sampler {
88
96
  ...(memory.psi ? { psi: memory.psi } : {}),
89
97
  };
90
98
  families.add('memory');
91
- if (memory.pressureLevel) observations.push({ key: 'memory.pressure', family: 'memory', def: alarm('memory.pressure'), value: memory.pressureLevel, met: { warn: memory.pressureLevel !== 'normal', crit: memory.pressureLevel === 'critical', clears: memory.pressureLevel === 'normal' }, summary: `kernel memory pressure ${memory.pressureLevel}` });
92
- if (memory.freePct != null) observations.push({ key: 'memory.free', family: 'memory', def: alarm('memory.free'), value: memory.freePct, unit: '%', summary: `memory free ${memory.freePct}%` });
93
- if (rate('swapouts') != null) observations.push({ key: 'memory.swapping', family: 'memory', def: alarm('memory.swapping'), value: rate('swapouts'), unit: 'pages/s', summary: `swapping out ${round(rate('swapouts'))} pages/s` });
94
- if (swapped30 != null) observations.push({ key: 'memory.swapped30m', family: 'memory', def: alarm('memory.swapped30m'), value: swapped30, unit: '% of RAM', summary: `${round(swapped30, 2)}% of RAM swapped out in the last ${readings.memory.swappedWindowMin} min` });
95
- if (rate('compressions') != null) observations.push({ key: 'memory.compressions', family: 'memory', def: alarm('memory.compressions'), value: rate('compressions'), unit: 'pages/s', summary: `${round(rate('compressions'), 0)} compressions/s` });
99
+ // A metric this platform does not have (memory.absent) is left out of judging: never raised, never unknown. Any
100
+ // other the platform gave no value for this cycle is unknown (null), never left out: left out, it would clear.
101
+ const absent = new Set(memory.absent ?? []);
102
+ const judged = (o) => { if (!absent.has(o.key ?? o.unreadKey)) observations.push(o); };
103
+ if (!memory.pressureLevel) judged({ unreadKey: 'memory.pressure', family: 'memory', why: 'the kernel memory pressure was not read this cycle' });
104
+ else judged({ key: 'memory.pressure', family: 'memory', def: alarm('memory.pressure'), value: memory.pressureLevel, met: { warn: memory.pressureLevel !== 'normal', crit: memory.pressureLevel === 'critical', clears: memory.pressureLevel === 'normal' }, summary: `kernel memory pressure ${memory.pressureLevel}` });
105
+ judged({ key: 'memory.free', family: 'memory', def: alarm('memory.free'), value: memory.freePct, unit: '%', summary: `memory free ${memory.freePct}%` });
106
+ judged({ key: 'memory.swapping', family: 'memory', def: alarm('memory.swapping'), value: rate('swapouts'), unit: 'pages/s', summary: `swapping out ${round(rate('swapouts'))} pages/s` });
107
+ judged({ key: 'memory.swapped30m', family: 'memory', def: alarm('memory.swapped30m'), value: swapped30, unit: '% of RAM', summary: `${round(swapped30, 2)}% of RAM swapped out in the last ${readings.memory.swappedWindowMin} min` });
108
+ judged({ key: 'memory.compressions', family: 'memory', def: alarm('memory.compressions'), value: rate('compressions'), unit: 'pages/s', summary: `${round(rate('compressions'), 0)} compressions/s` });
96
109
  });
97
110
 
98
111
  // ---- processes, renderers, daemons, writers ------------------------------------------------------
@@ -100,7 +113,12 @@ export class Sampler {
100
113
  if (on.has('processes')) await attempt('processes', async () => {
101
114
  const [rows, rus] = await Promise.all([
102
115
  this.reader.processes(),
103
- on.has('process-footprint') ? this.reader.rusage().catch((error) => { errors['process-footprint'] = error.message; return new Map(); }) : new Map(),
116
+ on.has('process-footprint') ? this.reader.rusage().catch((error) => {
117
+ errors['process-footprint'] = error.message;
118
+ // without footprints no growth is read: every growth alarm is unknown this cycle, not cleared
119
+ for (const prefix of ['process.growth:', 'renderer.growth:']) observations.push({ unreadPrefix: prefix, family: 'process', why: `process footprints could not be read: ${error.message}` });
120
+ return new Map();
121
+ }) : new Map(),
104
122
  ]);
105
123
  if (this.platform === 'darwin' && now - this.topFoot.at >= cfg.footprintTopEverySec * 1000) {
106
124
  this.topFoot = { at: now, map: await this.reader.topFootprints().catch((error) => { errors.daemons = error.message; return this.topFoot.map; }) };
@@ -145,6 +163,14 @@ export class Sampler {
145
163
  families.add('process');
146
164
  const daemonNames = new Set(cfg.daemons?.[this.platform] ?? []);
147
165
  const isDaemon = (r) => daemonNames.has(base(r.command));
166
+ // What this read covers, so a raised alarm it no longer shows clears only on that (thresholds.mjs judge): the
167
+ // process list is complete (a pid not in it is gone), and each check names the processes it judged on a value.
168
+ const pids = new Set(rows.map((r) => r.pid));
169
+ observations.push({ enumerated: 'process', family: 'process', by: 'pid', entities: pids });
170
+ const cpuJudged = (renderer) => new Set(byCpu.filter((r) => !isDaemon(r) && !AGENT.test(base(r.command)) && RENDERER.test(r.command) === renderer).map((r) => r.pid));
171
+ observations.push({ covered: 'process.cpu:', family: 'process', entities: cpuJudged(false) }, { covered: 'renderer.cpu:', family: 'process', entities: cpuJudged(true) });
172
+ const grown = (renderer) => new Set(rows.filter((r) => r.growthMBPerMin != null && RENDERER.test(r.command) === renderer).map((r) => r.pid));
173
+ observations.push({ covered: 'process.growth:', family: 'process', entities: grown(false) }, { covered: 'renderer.growth:', family: 'process', entities: grown(true) });
148
174
  // Every process down to the lower clear line is observed, so a raised alarm holds through a brief dip
149
175
  // instead of clearing and raising again (hysteresis); one that falls below it is no longer watched.
150
176
  for (const r of byCpu) {
@@ -159,8 +185,9 @@ export class Sampler {
159
185
  // observed; a raised one holds until its growth falls under a quarter of the line, so a leak that pauses
160
186
  // between samples is one alarm, not a raise and a clear each minute.
161
187
  for (const r of rows) {
162
- if (r.growthMBPerMin == null) continue;
163
188
  const renderer = RENDERER.test(r.command);
189
+ // a process whose growth has no reading this cycle (no footprint, or no earlier one) is unknown, not cleared
190
+ if (r.growthMBPerMin == null) { observations.push({ unreadKey: `${renderer ? 'renderer' : 'process'}.growth:${r.pid}`, family: 'process', why: 'no footprint reading to measure its growth this cycle' }); continue; }
164
191
  const tab = renderer ? this.rendererTabs?.get(r.pid) ?? null : null;
165
192
  const def = alarm(renderer ? 'renderer.growth' : 'process.growth');
166
193
  if (r.footprintBytes / MB < def.minMB) continue;
@@ -180,6 +207,7 @@ export class Sampler {
180
207
  orphanedHeadless: orphaned.map(view),
181
208
  };
182
209
  families.add('browser');
210
+ observations.push({ enumerated: 'browser', family: 'browser', by: 'pid', entities: pids }, { covered: 'browser.orphan:', family: 'browser', entities: pids });
183
211
  for (const r of orphaned) observations.push({ key: `browser.orphan:${r.pid}`, family: 'browser', def: alarm('browser.orphan'), value: 1, met: { warn: true, crit: false, clears: false }, summary: `headless browser ${r.pid} left running with launchd as its parent`, detail: view(r) });
184
212
  }
185
213
 
@@ -187,6 +215,8 @@ export class Sampler {
187
215
  const daemons = rows.filter(isDaemon);
188
216
  readings.daemons = daemons.sort((a, b) => (b.cpuPct ?? 0) - (a.cpuPct ?? 0)).map((r) => ({ name: base(r.command), ...view(r) }));
189
217
  families.add('daemon');
218
+ observations.push({ enumerated: 'daemon', family: 'daemon', by: 'pid', entities: pids },
219
+ { covered: 'daemon.cpu:', family: 'daemon', entities: new Set(daemons.filter((r) => r.cpuPct != null).map((r) => `${base(r.command)}:${r.pid}`)) });
190
220
  const footDef = alarm('daemon.footprint');
191
221
  for (const r of daemons) {
192
222
  const name = base(r.command);
@@ -200,6 +230,7 @@ export class Sampler {
200
230
  const writers = rows.filter((r) => r.writeBps > 0).sort((a, b) => b.writeBps - a.writeBps).slice(0, 5);
201
231
  readings.diskWriters = writers.map((r) => ({ ...view(r), writeMBps: round(r.writeBps / MB, 1) }));
202
232
  families.add('disk-writer');
233
+ observations.push({ enumerated: 'disk-writer', family: 'disk-writer', by: 'pid', entities: pids }, { covered: 'disk.writer:', family: 'disk-writer', entities: new Set(rows.filter((r) => r.writeBps != null).map((r) => r.pid)) });
203
234
  for (const r of rows) if (r.writeBps != null && r.writeBps / MB > alarm('disk.writer').warn * 0.5) observations.push({ key: `disk.writer:${r.pid}`, family: 'disk-writer', def: alarm('disk.writer'), value: r.writeBps / MB, unit: 'MB/s', summary: `process ${r.pid} writing ${round(r.writeBps / MB, 1)} MB/s`, detail: view(r) });
204
235
  }
205
236
 
@@ -276,6 +307,8 @@ export class Sampler {
276
307
  });
277
308
  readings.tcp = { states, timeWait: states.TIME_WAIT ?? 0, range: { ...range, size }, timeWaitSec, ephemeral, timeWaitByTarget };
278
309
  families.add('tcp');
310
+ // every socket was read: an address or a target with none has none in use (judged, under its line)
311
+ observations.push({ covered: 'tcp.ephemeral:', family: 'tcp', all: true }, { covered: 'tcp.timeWaitTarget:', family: 'tcp', all: true });
279
312
  for (const e of ephemeral) observations.push({ key: `tcp.ephemeral:${e.addr}`, family: 'tcp', def: alarm('tcp.ephemeral'), value: e.pct, unit: '% of ports', summary: `${e.used} of ${size} ephemeral ports in use on ${e.addr}` });
280
313
  for (const t of timeWaitByTarget) observations.push({ key: `tcp.timeWaitTarget:${t.target}`, family: 'tcp', def, value: t.pct, unit: '% of ports', summary: `${t.count} TIME_WAIT against ${t.target.startsWith(':') ? `listener ${t.target}${t.exited ? ' (listener has exited; its TIME_WAITs are draining)' : ''}` : t.target}${t.listener ? ` (pid ${t.listener.pid})` : ''}; range ${size}, TIME_WAIT lasts ${timeWaitSec} s`, detail: t });
281
314
  });
@@ -295,8 +328,8 @@ export class Sampler {
295
328
  for (const o of this.slow.observations) if (on.has(SLOW_FEATURE[o.family])) observations.push(o);
296
329
  for (const f of this.slow.families ?? []) if (on.has(SLOW_FEATURE[f])) families.add(f);
297
330
 
298
- // A feature turned off clears its raised alarms: its family is judged with nothing observed.
299
- for (const [family, feature] of Object.entries(FAMILY_FEATURE)) if (!on.has(feature)) families.add(family);
331
+ // A feature turned off clears its raised alarms: a positive observation that its family is no longer watched.
332
+ for (const [family, feature] of Object.entries(FAMILY_FEATURE)) if (!on.has(feature)) observations.push({ off: family, family });
300
333
 
301
334
  await this.#sessions(readings, observations);
302
335
  this.prev = { at: now, memory, io: this.io ?? null };
@@ -306,13 +339,25 @@ export class Sampler {
306
339
  async #slow(on, slow, errors, now) {
307
340
  const cfg = this.config;
308
341
  const alarm = (name) => cfg.alarms[name];
309
- const attempt = async (id, fn) => { try { await fn(); } catch (error) { errors[id] = error.message; } };
342
+ const attempt = async (id, fn) => { try { await fn(); } catch (error) { errors[id] = error.message; slow.observations.push(...unreadFamilies(id, error.message)); } };
310
343
 
311
344
  if (on.has('disk')) await attempt('disk', async () => {
312
345
  const def = alarm('disk.free');
313
346
  const fill = alarm('disk.filling');
314
347
  const vols = await this.reader.volumes();
315
348
  slow.families.add('disk');
349
+ // Every mount was read only when the listing finished and read each one: then one not listed is gone. A listing
350
+ // that did not (df failed, a mount's statfs failed) enumerates nothing, so a hung volume's alarm holds, unknown.
351
+ // Mounts this user may not stat (EACCES): named in the reading. They still exist, so they stay in the enumeration
352
+ // (never gone): an alarm raised on one holds, unknown, with why; one never raised costs nothing.
353
+ const denied = vols.denied ?? [];
354
+ if (denied.length) slow.readings.diskDenied = denied;
355
+ for (const mount of denied) for (const prefix of ['disk.free:', 'disk.filling:']) slow.observations.push({ unreadPrefix: `${prefix}${mount}`, family: 'disk', why: `${mount} can no longer be read by this user (statfs: EACCES)` });
356
+ if (vols.unread) for (const prefix of ['disk.free:', 'disk.filling:']) slow.observations.push({ unreadPrefix: prefix, family: 'disk', why: vols.unread });
357
+ else slow.observations.push({ enumerated: 'disk', family: 'disk', by: 'suffix', entities: new Set([...vols.map((v) => v.mount), ...denied]) });
358
+ // a mount's filling is judged once it has a rate (10 min of readings)
359
+ const rated = new Set();
360
+ slow.observations.push({ covered: 'disk.filling:', family: 'disk', entities: rated });
316
361
  slow.readings.disk = vols.map((v) => {
317
362
  const ring = this.diskRing.get(v.mount) ?? [];
318
363
  ring.push([now, v.freeBytes]);
@@ -322,6 +367,7 @@ export class Sampler {
322
367
  const hours = (now - t0) / 3_600_000;
323
368
  const perHour = hours >= 10 / 60 ? (f0 - v.freeBytes) / hours : null; // bytes consumed per hour
324
369
  const hoursToFull = perHour > 0 ? v.freeBytes / perHour : null;
370
+ if (perHour != null) rated.add(v.mount);
325
371
  const freePct = (v.freeBytes / v.totalBytes) * 100;
326
372
  const per = cfg.volumes?.[v.mount] ?? {};
327
373
  const row = { mount: v.mount, freeGB: round(v.freeBytes / GB), totalGB: round(v.totalBytes / GB), freePct: round(freePct), fillGBPerHour: perHour == null ? null : round(perHour / GB, 2), hoursToFull: round(hoursToFull) };
@@ -338,6 +384,8 @@ export class Sampler {
338
384
  const list = cfg.services ?? [];
339
385
  const jobs = await this.reader.services(list.map((s) => s.label));
340
386
  slow.families.add('service');
387
+ // every configured service was read (one no longer configured is gone)
388
+ slow.observations.push({ enumerated: 'service', family: 'service', by: 'suffix', entities: new Set(list.map((s) => s.label)) });
341
389
  slow.readings.services = list.map((s) => {
342
390
  const job = jobs.get(s.label);
343
391
  let logAgeMin = null;
@@ -393,7 +441,17 @@ export class Sampler {
393
441
  const cdef = alarm('supercode.connector');
394
442
  const link = connector?.enrolled ? (connector.running ? (connection == null ? null : connection === 'online') : false) : true;
395
443
  slow.observations.push({ key: 'supercode.daemon', family: 'supercode', def: cdef, value: described ? 0 : 1, met: { warn: !described, crit: false, clears: Boolean(described) }, summary: described ? 'machine daemon answering' : `machine daemon did not answer describe (${describe.error || 'no answer'})` });
396
- if (connector && link !== null) slow.observations.push({ key: 'supercode.link', family: 'supercode', def: cdef, value: link ? 0 : 1, met: { warn: !link, crit: false, clears: link }, summary: link ? 'connector linked to its server' : `connector ${connector.running ? `running but the server sees this machine ${connection}` : 'not running'}` });
444
+ // the link is read from two sources (the connector's status, the server's machine list): either unreadable leaves it
445
+ // unknown, which holds a raised alarm and clears nothing (a raised link alarm once cleared while the link was down)
446
+ if (!data) slow.observations.push({ unreadKey: 'supercode.link', family: 'supercode', why: `the connector's status could not be read: ${status.error || 'no answer'}` });
447
+ else if (connector && link === null) slow.observations.push({ unreadKey: 'supercode.link', family: 'supercode', why: `the server's view of this machine could not be read: ${machines.ok === false ? machines.error || 'no answer' : listed ? 'its machine record names no connection' : 'this machine is not in the machine list'}` });
448
+ // a status that names no connector says nothing of the link: unknown, said, never left out
449
+ else if (!connector) slow.observations.push({ unreadKey: 'supercode.link', family: 'supercode', why: 'the connector\'s status names no connector' });
450
+ else slow.observations.push({ key: 'supercode.link', family: 'supercode', def: cdef, value: link ? 0 : 1, met: { warn: !link, crit: false, clears: link }, summary: link ? 'connector linked to its server' : `connector ${connector.running ? `running but the server sees this machine ${connection}` : 'not running'}` });
451
+ // launches the daemon could not list are unknown: a raised launch alarm holds
452
+ if (!Array.isArray(rows)) for (const prefix of ['supercode.launch:', 'supercode.launchFailed:']) slow.observations.push({ unreadPrefix: prefix, family: 'supercode', why: `the daemon's launches could not be listed: ${launches.error || 'no answer'}` });
453
+ // the launch list was read: every launch in it was judged stuck or not, failed or not (one not shown is neither)
454
+ if (Array.isArray(rows)) slow.observations.push({ covered: 'supercode.launch:', family: 'supercode', all: true }, { covered: 'supercode.launchFailed:', family: 'supercode', all: true });
397
455
  for (const r of stuck) slow.observations.push({ key: `supercode.launch:${r.key}`, family: 'supercode', def: { forSec: 0 }, value: 1, met: { warn: true, crit: false, clears: false }, summary: r.status === 'issued' ? `launch ${r.key} (${r.kind ?? 'pane'}) issued ${Math.round((now - r.at) / 60_000)} min ago and never became ready` : `launch ${r.key} is live but its first input was not delivered: ${r.inputError}` });
398
456
  for (const r of failed) slow.observations.push({ key: `supercode.launchFailed:${r.key}`, family: 'supercode', def: { forSec: 0 }, value: 1, met: { warn: true, crit: false, clears: false }, summary: `launch ${r.key} (${r.kind ?? 'pane'}) failed at ${new Date(r.at).toISOString()}` });
399
457
  });
@@ -428,7 +486,10 @@ export class Sampler {
428
486
  slow.readings.browserTabs = { count: tabs.length, byHolder, tabs: rows };
429
487
  slow.families.add('tabs');
430
488
  const def = alarm('browser.tabs');
489
+ // Said either way, never left out: with no line configured (the default) the count was read and is past no line,
490
+ // a positive observation that clears an alarm raised under a line since removed.
431
491
  if (def.warn != null) slow.observations.push({ key: 'browser.tabs', family: 'tabs', def, value: tabs.length, unit: 'tabs', summary: `${tabs.length} browser tabs open` });
492
+ else slow.observations.push({ key: 'browser.tabs', family: 'tabs', def, value: tabs.length, unit: 'tabs', met: { warn: false, crit: false, clears: true }, summary: `${tabs.length} browser tabs open (no line configured)` });
432
493
  });
433
494
 
434
495
  if (on.has('power')) await attempt('power', async () => {
@@ -437,7 +498,8 @@ export class Sampler {
437
498
  const level = /pressure level:\s*(\w+)/i.exec(r.stdout)?.[1] ?? null;
438
499
  slow.readings.power = { thermalPressure: level };
439
500
  slow.families.add('power');
440
- if (level) slow.observations.push({ key: 'power.thermal', family: 'power', def: alarm('power.thermal'), value: level, met: { warn: level !== 'Nominal', crit: /Heavy|Trapping|Sleeping/.test(level), clears: level === 'Nominal' }, summary: `thermal pressure ${level}` });
501
+ if (!level) slow.observations.push({ unreadKey: 'power.thermal', family: 'power', why: 'the thermal pressure was not read this cycle' });
502
+ else slow.observations.push({ key: 'power.thermal', family: 'power', def: alarm('power.thermal'), value: level, met: { warn: level !== 'Nominal', crit: /Heavy|Trapping|Sleeping/.test(level), clears: level === 'Nominal' }, summary: `thermal pressure ${level}` });
441
503
  });
442
504
  }
443
505
 
@@ -5,6 +5,7 @@ export const DEFAULTS = {
5
5
  slowIntervalSec: 60, // disk space, services, the connector, tabs
6
6
  footprintTopEverySec: 300, // macOS: every user's largest footprints through top (about 2 s of top per run)
7
7
  repeatMin: 60, // a raised alarm mails again this often (netdata's repeat cap)
8
+ unknownMin: 15, // a reading unknown this long (raised or not) is mailed to the maintainers, and again each repeatMin
8
9
  historyHours: 24,
9
10
  daemons: {
10
11
  darwin: ['dasd', 'fseventsd', 'mds', 'mds_stores', 'mdworker_shared', 'WindowServer', 'kernel_task', 'backupd', 'softwareupdated', 'photoanalysisd', 'cloudd', 'bird', 'nsurlsessiond', 'trustd', 'syspolicyd', 'XprotectService', 'logd', 'runningboardd', 'launchservicesd', 'coreduetd', 'spotlightknowledged'],
@@ -48,7 +49,8 @@ export function merge(base, over) {
48
49
 
49
50
  /** Which levels a numeric value meets under a threshold definition. */
50
51
  export function meets(value, def) {
51
- if (value == null || Number.isNaN(value)) return { warn: false, crit: false, clears: true };
52
+ // no value is no reading: unknown, which neither raises nor clears (D190 class 1)
53
+ if (value == null || Number.isNaN(value)) return { warn: false, crit: false, clears: false, unknown: true };
52
54
  const past = (line) => line != null && (def.below ? value < line : value > line);
53
55
  const crit = past(def.crit);
54
56
  const warn = crit || past(def.warn);
@@ -59,16 +61,56 @@ export function meets(value, def) {
59
61
  const RANK = { none: 0, warn: 1, crit: 2 };
60
62
 
61
63
  /**
62
- * One step of every alarm. `observations`: [{ key, family, def, value, met?, unit, summary, detail }] for the
63
- * families read this cycle. A key of an evaluated family that was not observed has cleared (its entity is gone).
64
- * Returns the transitions; `state` (key → record) is updated in place.
64
+ * One step of every alarm. A raised alarm clears only on a positive observation (D190 class 1, held structurally:
65
+ * no read site has to say it failed for its alarm to hold):
66
+ * - its value, read this cycle, is past its clear line;
67
+ * - the read that judges it covered it with a real value and found it under its line (`{ covered: prefix, family,
68
+ * entities | all }`: every key `prefix + entity` was evaluated, so one not observed is under its line);
69
+ * - a complete enumeration of its family no longer has its entity (`{ enumerated: family, by: 'pid' | 'suffix',
70
+ * entities }`: the process list, the mounts, the configured services);
71
+ * - its feature is off (`{ off: family }`).
72
+ * Anything else not observed is unknown: the alarm holds, saying since when and why (`{ unreadFamily | unreadKey |
73
+ * unreadPrefix, why }` give the why where a site knows it; otherwise it is "not observed this cycle").
74
+ * `observations` also carry the readings: [{ key, family, def, value, met?, unit, summary, detail }].
75
+ * Every unknown (raised or not: a key or a family that could not be read) is kept in `unknowns` (key → { why,
76
+ * since }) for its notification (probe.mjs); one read again leaves it.
77
+ * Returns the transitions; `state` (key → record) and `unknowns` are updated in place.
65
78
  */
66
- export function judge(state, observations, families, now) {
79
+ export function judge(state, observations, families, now, unknowns = {}) {
67
80
  const transitions = [];
68
81
  const seen = new Set();
82
+ const unreadFamilies = new Map(), unreadKeys = new Map(), unreadPrefixes = [], covered = [], enumerated = new Map(), off = new Set();
69
83
  for (const o of observations) {
84
+ if (o.unreadFamily) unreadFamilies.set(o.unreadFamily, o.why);
85
+ else if (o.unreadKey) unreadKeys.set(o.unreadKey, { family: o.family, why: o.why });
86
+ else if (o.unreadPrefix) unreadPrefixes.push(o);
87
+ else if (o.covered) covered.push(o);
88
+ else if (o.enumerated) enumerated.set(o.enumerated, o);
89
+ else if (o.off) off.add(o.off);
90
+ }
91
+ const why = (key, family) => unreadFamilies.get(family) ?? unreadKeys.get(key)?.why ?? unreadPrefixes.find((p) => p.family === family && key.startsWith(p.unreadPrefix))?.why ?? null;
92
+ // an entity is a pid (a number) or a name: a key's suffix matches either form
93
+ const holds = (set, entity) => set.has(entity) || (/^\d+$/.test(entity) && set.has(Number(entity)));
94
+ const under = (key, family) => covered.some((c) => c.family === family && key.startsWith(c.covered) && (c.all || (c.entities && holds(c.entities, key.slice(c.covered.length)))));
95
+ const gone = (key, family) => {
96
+ const e = enumerated.get(family); if (!e) return false;
97
+ const entity = e.by === 'pid' ? Number(key.slice(key.lastIndexOf(':') + 1)) : key.slice(key.indexOf(':') + 1);
98
+ return !e.entities.has(entity);
99
+ };
100
+ const hold = (key, reason) => {
101
+ const s = state[key];
102
+ if (s) Object.assign(s, { unread: String(reason), unreadAt: s.unreadAt ?? now });
103
+ unknowns[key] = { family: s?.family ?? null, why: String(reason), since: unknowns[key]?.since ?? now };
104
+ };
105
+ // what a site said it could not read is unknown whether or not it was ever raised (its notification: probe.mjs)
106
+ const read = new Set();
107
+ for (const [family, reason] of unreadFamilies) { const key = `family:${family}`; unknowns[key] = { family, why: reason, since: unknowns[key]?.since ?? now }; read.add(key); }
108
+ for (const [key, { family, why: reason }] of unreadKeys) { unknowns[key] = { family, why: reason, since: unknowns[key]?.since ?? now }; read.add(key); }
109
+ for (const o of observations) {
110
+ if (!o.key) continue;
70
111
  seen.add(o.key);
71
112
  const m = o.met ?? meets(o.value, o.def);
113
+ if (m.unknown) { hold(o.key, `${o.key}: no value read this cycle`); read.add(o.key); continue; }
72
114
  const s = state[o.key] ?? (state[o.key] = { key: o.key, family: o.family, level: 'none', warnSince: null, critSince: null, raisedAt: null, lastMailAt: null });
73
115
  s.warnSince = m.warn ? (s.warnSince ?? now) : null;
74
116
  s.critSince = m.crit ? (s.critSince ?? now) : null;
@@ -79,7 +121,7 @@ export function judge(state, observations, families, now) {
79
121
  else if (m.warn && now - s.warnSince >= warnFor) target = 'warn';
80
122
  // Hysteresis: a raised alarm holds at warn until the value is past its clear line.
81
123
  if (target === 'none' && s.level !== 'none' && !m.clears) target = 'warn';
82
- Object.assign(s, { value: o.value, unit: o.unit ?? null, summary: o.summary, detail: o.detail ?? null, at: now });
124
+ Object.assign(s, { value: o.value, unit: o.unit ?? null, summary: o.summary, detail: o.detail ?? null, at: now, unread: null, unreadAt: null });
83
125
  if (target !== s.level) {
84
126
  transitions.push({ key: o.key, from: s.level, to: target, value: o.value, unit: o.unit ?? null, summary: o.summary, detail: o.detail ?? null });
85
127
  if (RANK[target] > RANK[s.level] && s.level === 'none') s.raisedAt = now;
@@ -88,16 +130,27 @@ export function judge(state, observations, families, now) {
88
130
  }
89
131
  }
90
132
  for (const [key, s] of Object.entries(state)) {
91
- if (seen.has(key) || !families.has(s.family)) continue;
92
- // An alarm restored from before a restart is not cleared by the first samples, which have no rates yet.
93
- if (s.restoredUntil && now < s.restoredUntil) continue;
94
- if (s.level !== 'none') transitions.push({ key, from: s.level, to: 'none', value: null, summary: `${s.summary} — cleared: no longer over its line, or the process exited`, detail: s.detail, gone: true });
133
+ if (seen.has(key)) continue;
134
+ // a key not observed clears only on a positive observation; anything else is unknown
135
+ const cleared = off.has(s.family) ? 'its feature is off' : under(key, s.family) ? 'under its line' : gone(key, s.family) ? 'it is gone' : null;
136
+ if (!cleared) {
137
+ if (s.level === 'none') { delete state[key]; continue; } // nothing raised to hold
138
+ // An alarm restored from before a restart waits out the first samples, which have no rates yet.
139
+ if (s.restoredUntil && now < s.restoredUntil) continue;
140
+ hold(key, why(key, s.family) ?? `not observed this cycle: no complete read of its ${s.family} family covered it`);
141
+ read.add(key);
142
+ continue;
143
+ }
144
+ if (s.level !== 'none') transitions.push({ key, from: s.level, to: 'none', value: null, summary: `${s.summary} — cleared: ${cleared}`, detail: s.detail, gone: cleared === 'it is gone' });
95
145
  delete state[key];
96
146
  }
147
+ // an unknown read again (or cleared) this cycle leaves the list
148
+ for (const key of Object.keys(unknowns)) if (!read.has(key)) delete unknowns[key];
97
149
  return transitions;
98
150
  }
99
151
 
100
152
  export function raised(state) {
101
153
  return Object.values(state).filter((s) => s.level !== 'none').sort((a, b) => RANK[b.level] - RANK[a.level] || a.key.localeCompare(b.key))
102
- .map(({ key, family, level, value, unit, summary, detail, raisedAt }) => ({ key, family, level, value, unit, summary, detail, since: raisedAt ? new Date(raisedAt).toISOString() : null }));
154
+ .map(({ key, family, level, value, unit, summary, detail, raisedAt, unread, unreadAt }) => ({ key, family, level, value, unit, summary, detail, since: raisedAt ? new Date(raisedAt).toISOString() : null,
155
+ ...(unread ? { unknown: { why: unread, since: new Date(unreadAt).toISOString() } } : {}) }));
103
156
  }
package/lib/win32.mjs CHANGED
@@ -5,7 +5,9 @@ import { run } from './run.mjs';
5
5
 
6
6
  export async function memory() {
7
7
  const total = totalmem();
8
- return { pressureLevel: null, totalBytes: total, freePct: Math.round((freemem() / total) * 100), swapUsedBytes: null, swapTotalBytes: null, counters: {}, pageSize: 4096 };
8
+ // what Node gives no reading of on Windows: left out of judging, never unknown
9
+ return { pressureLevel: null, totalBytes: total, freePct: Math.round((freemem() / total) * 100), swapUsedBytes: null, swapTotalBytes: null, counters: {}, pageSize: 4096,
10
+ absent: ['memory.pressure', 'memory.swapping', 'memory.swapped30m', 'memory.compressions'] };
9
11
  }
10
12
 
11
13
  function splitAddress(text) {
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@volter/supercode-health",
3
- "version": "0.1.3",
3
+ "version": "0.1.5",
4
4
  "type": "module",
5
5
  "description": "Optional machine pack: a cheap, deterministic probe of a machine's vitals that mails the machine's maintainer when a threshold is crossed. Each feature sits behind its own permission barrier.",
6
6
  "license": "MIT",