@volter/supercode-health 0.1.21 → 0.1.22

package/README.md

@@ -42,6 +42,24 @@ still-raised alarm mails again every `repeatMin` (60). Mail comes from
 it, within its bounds, and `supercode message waiting` / `supercode message expired` list
 what it has not delivered.
 
+## fseventsd on macOS
+
+fseventsd logs `event table has grown too large` once, when its path table reaches its maximum; after that it may
+free nothing, and its memory then grows with every new file until it is restarted (26 GB and 23 GB on two fleet
+Macs). The probe reads that line from the unified log every two minutes (`/usr/bin/log show`, the window since its
+last read, asked by the pid launchd runs and its system image path, never by a process's name; unknown, and said so,
+where the log cannot be read) and raises `fseventsd.overflow:<pid>` at crit with
+what to look at and the restart. It clears when that daemon is gone. `daemon.footprint:fseventsd:<pid>` warns past
+192 MB and is critical past 256 MB (3–60 MB is normal).
+
+The probe never restarts anything. It also reads whether the fseventsd guard is installed and running: a root job
+outside this pack that performs that restart, installed only by a person's own sudo
+([its README](../../packaging/npm/guard/README.md)). The reading comes from the world-readable files the guard
+leaves in `/Library/dev.volter.supercode-health`. `supercode health status` always prints it: not installed (with the
+line a person would run), installed with its last run, installed and silent, or installed and unable to read the
+log, and its last restart if it ever made one. Any state but the second raises `fseventsd.guard` at warn, mailed when it begins and when it ends, never as a
+reminder.
+
 ## Failures that keep happening
 
 The `supercode` feature also reads the machine's own records of failure each minute: the slow log

package/bin/health.mjs

@@ -38,7 +38,7 @@ if (values.help || verb === 'help') {
 } else if (verb === 'once') {
   const probe = new Probe({ machine: values.machine, dir: `${dir}/once-${process.pid}` });
   await probe.init();
-  await probe.sampler.sample(probe.features(), Date.now(), { wait: true });
+  await probe.sampler.sample(probe.features(), Date.now());
   await new Promise((r) => setTimeout(r, 5_000));
   const now = Date.now();
   const { readings, observations, families, errors } = await probe.sampler.sample(probe.features(), now, { wait: true });

package/lib/darwin.mjs

@@ -1,6 +1,6 @@
 // macOS readers. Every one runs unprivileged; each returns raw cumulative values and leaves rates to derive.mjs.
-import { existsSync } from 'node:fs';
-import { delimiter, isAbsolute, join } from 'node:path';
+import { closeSync, existsSync, fstatSync, openSync, readFileSync, readSync, statSync } from 'node:fs';
+import { delimiter, dirname, isAbsolute, join, resolve } from 'node:path';
 import { fileURLToPath } from 'node:url';
 import { run, clockSeconds, topBytes } from './run.mjs';
 
@@ -181,5 +181,125 @@ export async function services(labels = []) {
   return map;
 }
 
+const FSE_SERVICE = 'system/com.apple.fseventsd';
+const FSE_IMAGE = '/System/Library/Frameworks/CoreServices.framework/';
+
+/**
+ * The fseventsd launchd runs as its system service: { pid, startedAt (ms, from ps's elapsed time) }, or null when it
+ * runs none. The pid is launchd's own record (`launchctl print` answers without root), never a process's name.
+ */
+export async function fseventsd({ signal } = {}) {
+  const r = await run('/bin/launchctl', ['print', FSE_SERVICE], { timeoutMs: 5000, signal });
+  if (!r.ok) throw new Error(r.error);
+  const pid = Number(/^\s*pid = (\d+)\s*$/m.exec(r.stdout)?.[1]);
+  if (!pid) return null;
+  const ps = await run('/bin/ps', ['-o', 'etime=', '-p', String(pid)], { timeoutMs: 5000, signal });
+  if (!ps.ok || !ps.stdout.trim()) throw new Error(`fseventsd ${pid} was not listed by ps: ${ps.error ?? 'no row'}`);
+  return { pid, startedAt: Date.now() - clockSeconds(ps.stdout.trim()) * 1000 };
+}
+
+/** One process's footprint as top reads it (phys_footprint, compressed pages included), unprivileged; about 1 s. */
+export async function footprintOf(pid, { signal } = {}) {
+  const r = await run('/usr/bin/top', ['-l', '1', '-pid', String(pid), '-stats', 'pid,mem'], { timeoutMs: 10_000, signal });
+  if (!r.ok) throw new Error(r.error);
+  for (const line of r.stdout.split('\n')) {
+    const m = /^\s*(\d+)\s+(\S+)\s*$/.exec(line);
+    if (m && +m[1] === pid) return topBytes(m[2]);
+  }
+  return null;
+}
+
+/** The records the unified log holds as the given fseventsd's own "event table has grown too large" line. Who wrote a
+ *  record is asked of the log (its pid and its image, which only the system can place on that sealed path), never
+ *  read out of text: any program may name itself fseventsd and print what it likes. The guard uses the same rule. */
+export const overflowPredicate = (pid) => `processIdentifier == ${Number(pid)} AND processImagePath BEGINSWITH "${FSE_IMAGE}" AND composedMessage CONTAINS "grown too large"`;
+const pad = (n) => String(n).padStart(2, '0');
+const logTime = (ms) => { const d = new Date(ms); return `${d.getFullYear()}-${pad(d.getMonth() + 1)}-${pad(d.getDate())} ${pad(d.getHours())}:${pad(d.getMinutes())}:${pad(d.getSeconds())}`; };
+// `log show` takes local wall time: beside a clock change an hour of it is ambiguous, so the window is an hour wider there
+const shifts = (ms) => new Date(ms - 3_600_000).getTimezoneOffset() !== new Date(ms + 3_600_000).getTimezoneOffset();
+
+/**
+ * That line for the daemon `pid`, read unprivileged (`log show` answers a member of the admin group; anyone else gets
+ * an error, which is thrown). One window: `lastMin` (the minutes up to now; about 2 s for 5 or 30 min) or `from`..`to`
+ * (ms). Each record is one line of JSON, checked again field by field; a read that did not end with the log's own
+ * `finished` record is thrown, never taken as empty. Answers [{ at (ISO, null when the time is not read), line }].
+ */
+export async function fseventsOverflow({ pid, lastMin, from, to, timeoutMs = 30_000, signal }) {
+  const window = lastMin != null ? ['--last', `${lastMin}m`] : ['--start', logTime(shifts(from) ? from - 3_600_000 : from), '--end', logTime(shifts(to) ? to + 3_600_000 : to)];
+  const r = await run('/usr/bin/log', ['show', ...window, '--predicate', overflowPredicate(pid), '--style', 'ndjson'], { timeoutMs, signal });
+  if (!r.ok) throw new Error(r.error);
+  const hits = [];
+  let finished = false;
+  for (const line of r.stdout.split('\n')) {
+    let record = null; try { record = JSON.parse(line); } catch { continue; }
+    if (record?.finished === 1) { finished = true; continue; }
+    if (record?.processID !== pid || !String(record.processImagePath ?? '').startsWith(FSE_IMAGE) || !String(record.eventMessage ?? '').includes('grown too large')) continue;
+    const at = new Date(String(record.timestamp ?? '').replace(' ', 'T').replace(/([+-]\d\d)(\d\d)$/, '$1:$2'));
+    hits.push({ at: Number.isNaN(at.getTime()) ? null : at.toISOString(), line: String(record.eventMessage).replace(/[^\x20-\x7e]/g, ' ').slice(0, 200) });
+  }
+  if (!finished) throw new Error(`log show did not finish its listing: ${(r.stderr || r.stdout).trim().slice(0, 200) || 'it printed nothing'}`);
+  return hits;
+}
+
+const GUARD_DIR = '/Library/dev.volter.supercode-health';
+const GUARD_PLIST = '/Library/LaunchDaemons/dev.volter.supercode-health.fseventsd-guard.plist';
+
+/**
+ * Where this install carries the guard's installer, or null. The guard ships in the release package
+ * (@volter/supercode/guard), so from this pack's own place: the release it is installed inside
+ * (<release>/node_modules/@volter/supercode-health), the release beside the package it is installed inside (a global
+ * @volter/supercode-teams next to @volter/supercode), or a checkout (packaging/npm/guard).
+ */
+function guardInstaller() {
+  const lib = dirname(fileURLToPath(import.meta.url));
+  const named = (dir) => { try { return JSON.parse(readFileSync(join(dir, 'package.json'), 'utf8')).name; } catch { return null; } };
+  const outer = resolve(lib, '..', '..', '..', '..'), repo = resolve(lib, '..', '..', '..');
+  const dirs = [outer, resolve(outer, '..', 'supercode')].filter((dir) => named(dir) === '@volter/supercode').map((dir) => join(dir, 'guard'));
+  if (existsSync(join(repo, 'packaging', 'npm', 'package.json.tmpl'))) dirs.push(join(repo, 'packaging', 'npm', 'guard'));
+  for (const dir of dirs) {
+    const file = join(dir, 'fseventsd-guard-install.sh');
+    if (existsSync(file) && existsSync(join(dir, 'fseventsd-guard.sh'))) return file;
+  }
+  return null;
+}
+
+/** The last `bytes` of a file as text, or null: the guard's log is read by its end, never whole. */
+function tail(file, bytes = 4096) {
+  let fd = null;
+  try {
+    fd = openSync(file, 'r');
+    const size = fstatSync(fd).size, buffer = Buffer.alloc(Math.min(size, bytes));
+    readSync(fd, buffer, 0, buffer.length, size - buffer.length);
+    return buffer.toString('utf8');
+  } catch { return null; } finally { if (fd != null) closeSync(fd); }
+}
+const quoted = (text) => `'${String(text).replace(/'/g, `'\\''`)}'`;
+
+/**
+ * The fseventsd guard (packaging/npm/guard: a root job outside this pack, which a person installs with their own
+ * sudo and which restarts an overflowed fseventsd past its line), read from the files it leaves, all world-readable:
+ * whether it is installed (its plist and its script, both root's), its last run and its last restart. `install` and
+ * `uninstall` are the lines a person runs, named for the status; the pack reads and runs nothing of the guard's.
+ */
+export function fseventsdGuard(critMB) {
+  const stat = (file) => { try { return statSync(file); } catch { return null; } };
+  const plist = stat(GUARD_PLIST), script = stat(`${GUARD_DIR}/fseventsd-guard.sh`);
+  const last = tail(`${GUARD_DIR}/fseventsd-guard.last`, 512)?.trim() || null, installer = guardInstaller();
+  const at = Date.parse(/^at=(\S+)/.exec(last ?? '')?.[1] ?? '');
+  // the log's last whole line (a tail may begin inside one)
+  const acts = tail(`${GUARD_DIR}/fseventsd-guard.log`)?.split('\n').filter(Boolean) ?? [];
+  return {
+    installed: Boolean(plist && script && plist.uid === 0 && script.uid === 0),
+    installedAt: plist ? new Date(plist.mtimeMs).toISOString() : null,
+    lastRunAt: Number.isNaN(at) ? null : new Date(at).toISOString(),
+    lastRun: last ? last.replace(/^at=\S+\s*/, '').replace(/[^\x20-\x7e]/g, ' ').slice(0, 300) : null,
+    // how many runs in a row could not read the log (the guard's own count)
+    unreadRuns: / line=unread /.test(` ${last} `) ? Number(/ unread_runs=(\d+)/.exec(last ?? '')?.[1] ?? 1) : 0,
+    lastAct: acts.at(-1)?.replace(/[^\x20-\x7e]/g, ' ').slice(0, 600) ?? null,
+    install: installer ? `sudo /bin/sh ${quoted(installer)}${critMB ? ` ${Math.round(critMB)}` : ''}` : null,
+    uninstall: installer ? `sudo /bin/sh ${quoted(installer)} uninstall` : null,
+  };
+}
+
 export const isRoot = () => process.getuid?.() === 0;
 export const powermetricsAvailable = () => existsSync('/usr/bin/powermetrics');

package/lib/features.mjs

@@ -8,7 +8,7 @@ export const FEATURES = [
   { id: 'process-footprint', barrier: 'none', platforms: ['darwin', 'linux'], reads: "this user's processes: phys_footprint and disk writes (libproc / /proc/<pid>/io)" },
   { id: 'all-users-footprint', barrier: 'root', platforms: ['darwin', 'linux'], reads: "every user's processes: footprint and disk writes" },
   { id: 'renderers', barrier: 'none', platforms: ['darwin', 'linux'], reads: 'browser renderers: hot, runaway; orphaned headless browsers' },
-  { id: 'daemons', barrier: 'none', platforms: ['darwin', 'linux'], reads: 'system daemons: CPU, and footprint of the largest' },
+  { id: 'daemons', barrier: 'none', platforms: ['darwin', 'linux'], reads: "system daemons: CPU, and footprint of the largest; macOS: fseventsd's event-table overflow line in the unified log" },
   { id: 'disk', barrier: 'none', platforms: ['darwin', 'linux', 'win32'], reads: 'free space per volume, fill rate, time to full' },
   { id: 'disk-io', barrier: 'none', platforms: ['darwin', 'linux'], reads: 'device throughput; top writers per process' },
   { id: 'tcp', barrier: 'none', platforms: ['darwin', 'linux', 'win32'], reads: 'sockets per state, ephemeral ports in use, TIME_WAIT by listener and its pid' },

package/lib/format.mjs

@@ -40,6 +40,8 @@ export function formatStatus(s, { all = false } = {}) {
     for (const t of r.browserTabs.tabs ?? []) out.push(`  ${t.url.slice(0, 70)} · ${t.owner}${who(t.server)}${t.renderers.length ? ` · renderer ${t.renderers.join(', ')}` : ''}`);
   }
   if (r.daemons) out.push(`daemons   ${r.daemons.filter((d) => (d.cpuPct ?? 0) >= 5 || d.footprintMB >= 512).map((d) => `${d.name} ${n(d.cpuPct, '%')} ${d.footprintMB} MB`).join(' · ') || 'all quiet'}`);
+  if (r.fseventsd) out.push(`fseventsd pid ${r.fseventsd.pid} (started ${r.fseventsd.startedAt}): event table ${r.fseventsd.eventTable}${r.fseventsd.overflowAt ? ` at ${r.fseventsd.overflowAt}` : ''} · log read to ${r.fseventsd.logReadTo}`);
+  if (r.fseventsdGuard) out.push(r.fseventsdGuard.state, ...(r.fseventsdGuard.acted ? [`  ${r.fseventsdGuard.acted}`] : []));
   if (r.disk) for (const d of r.disk) out.push(`disk      ${d.mount}: ${d.freeGB} GB free of ${d.totalGB} (${d.freePct}%)${d.fillGBPerHour != null ? ` · ${d.fillGBPerHour >= 0 ? 'filling' : 'freeing'} ${Math.abs(d.fillGBPerHour)} GB/h` : ''}${d.hoursToFull != null ? ` · full in ${d.hoursToFull} h` : ''}`);
   if (r.diskIO) out.push(`disk I/O  ${Object.entries(r.diskIO).map(([k, v]) => `${k} ${n(v.MBps, ' MB/s')}`).join(' · ')}`);
   if (r.diskWriters?.length) out.push(`writers   ${r.diskWriters.slice(0, 3).map((w) => `${w.pid} ${w.writeMBps} MB/s ${w.command.slice(0, 50)}${who(w)}`).join(' · ')}`);

package/lib/probe.mjs

@@ -167,7 +167,8 @@ export class Probe {
       net.push({ key, from: was.level, to: 'none', value: null, summary: t?.to === 'none' ? t.summary : `${was.summary} — cleared`, detail: t?.detail ?? was.detail });
     }
     const raisedNow = new Set(net.filter((t) => t.to !== 'none').map((t) => t.key));
-    const reminders = alarms.filter((a) => !raisedNow.has(a.key) && now - (this.state[a.key].lastMailAt ?? 0) >= repeatMs);
+    // A standing state (thresholds.mjs judge) is never reminded: its raise and its clear are its two mails.
+    const reminders = alarms.filter((a) => !raisedNow.has(a.key) && !this.state[a.key].standing && now - (this.state[a.key].lastMailAt ?? 0) >= repeatMs);
     // A reading unknown for unknownMin is the maintainers' to know (raised or not), and again each repeatMin until read.
     // Timed per kind (a key's prefix; a family's whole key): a kind is due once its earliest key has been unknown
     // unknownMin and the kind was last mailed repeatMin ago, and then every key of it is said. Keys that join a kind one
@@ -287,6 +288,8 @@ export class Probe {
   async run({ signal } = {}) {
     await this.init();
     this.log(`probe ${process.pid} started on ${this.machine} (${this.platform}); features on: ${this.features().filter((f) => f.on).map((f) => f.id).join(', ')}`);
+    // a probe told to stop ends the reads that run beside its loop, so no child of theirs outlives it
+    signal?.addEventListener('abort', () => this.sampler.stop(), { once: true });
     while (!signal?.aborted) {
       const started = Date.now();
       try { await this.cycle(started); } catch (error) { this.log(`cycle failed: ${error.stack ?? error.message}`); }

package/lib/run.mjs

@@ -1,10 +1,11 @@
 // One bounded child process. Every instrument goes through here: a reading that cannot finish in its budget
-// is an error the feature reports, never a hang of the probe.
+// is an error the feature reports, never a hang of the probe. `signal` ends the child with its caller (a probe that
+// stops leaves none behind).
 import { execFile } from 'node:child_process';
 
-export function run(file, args = [], { timeoutMs = 10_000, env, input, maxBuffer = 64 * 1024 * 1024 } = {}) {
+export function run(file, args = [], { timeoutMs = 10_000, env, input, signal, maxBuffer = 64 * 1024 * 1024 } = {}) {
   return new Promise((resolve) => {
-    const child = execFile(file, args, { timeout: timeoutMs, env, maxBuffer, windowsHide: true, encoding: 'utf8', shell: process.platform === 'win32' }, (error, stdout, stderr) => {
+    const child = execFile(file, args, { timeout: timeoutMs, env, maxBuffer, signal, windowsHide: true, encoding: 'utf8', shell: process.platform === 'win32' }, (error, stdout, stderr) => {
       if (error) resolve({ ok: false, stdout: stdout ?? '', stderr: stderr ?? '', error: error.killed ? `${file} timed out after ${timeoutMs} ms` : (error.code === 'ENOENT' ? `${file} not found` : (stderr || error.message).trim().slice(0, 300)) });
       else resolve({ ok: true, stdout, stderr });
     });

package/lib/sample.mjs

@@ -1,7 +1,7 @@
 // The sampler: reads every feature that is on, turns cumulative counters into rates against the previous
 // sample, and states each alarm's observation. It keeps only what a rate or a window needs between samples.
 import { cpus, homedir, loadavg } from 'node:os';
-import { existsSync, readFileSync, statSync } from 'node:fs';
+import { existsSync, readFileSync, renameSync, statSync, writeFileSync } from 'node:fs';
 import { run } from './run.mjs';
 import { daemonCall } from './mail.mjs';
 import { rendererOrigins, originOf, localPort } from './chrome.mjs';
@@ -26,6 +26,13 @@ const HARNESS = { claude: 'claude-code', codex: 'codex', gemini: 'gemini', goose
 const RENDERER = /Helper \(Renderer\)|--type=renderer/;
 const BROWSER_MAIN = /(Google Chrome|Chromium|chrome-headless-shell|chrome)(\s|$)/i;
 const LOOPBACK = /^(127\.|::1$|localhost$)/;
+// fseventsd's alarms say what they mean, what to look at and the one act that clears them (ADR 0006). The pack does
+// not perform the act: a person does, or the root guard a person installed (packaging/npm/guard, outside the pack).
+const FSE_LINE = 'event table has grown too large';
+const FSE_ACT = 'Restart it as root: `sudo launchctl kickstart -k system/com.apple.fseventsd` (or `sudo killall fseventsd`); launchd starts it again and its clients reconnect';
+const FSE_GROWTH = 'a footprint that rises and never falls back while the disk is quiet is that growth (it reached 26 GB on a fleet Mac)';
+// What a daemon past its own footprint line means, said in its alarm.
+const DAEMON_PAST = { fseventsd: `3–60 MB is normal. If it has logged "${FSE_LINE}" (the fseventsd.overflow alarm), its path table is full and may free nothing more: ${FSE_GROWTH}. ${FSE_ACT}` };
 /** The session an agent names on its own command line: `--resume <id>`, `--session-id <id>`, `-r <id>`, `codex resume <id>`. */
 function argvSession(command) {
   return /(?:--resume|--session-id|\s-r)[=\s]+([0-9a-f]{8}-[0-9a-f-]{27,})/.exec(command)?.[1]
@@ -34,7 +41,7 @@ function argvSession(command) {
 // The slow set's readings and alarm families, by the feature that produces them.
 const SLOW_FEATURE = { promises: 'promises', promise: 'promises', disk: 'disk', services: 'services', service: 'services', supercode: 'supercode', browserTabs: 'browser-tabs', tabs: 'browser-tabs', power: 'power' };
 // Every alarm family, by the feature that observes it.
-const FAMILY_FEATURE = { promise: 'promises', load: 'load', memory: 'memory', process: 'processes', browser: 'renderers', daemon: 'daemons', 'disk-writer': 'disk-io', files: 'files', tcp: 'tcp', disk: 'disk', service: 'services', supercode: 'supercode', tabs: 'browser-tabs', power: 'power' };
+const FAMILY_FEATURE = { promise: 'promises', load: 'load', memory: 'memory', process: 'processes', browser: 'renderers', daemon: 'daemons', fsevents: 'daemons', 'fsevents-guard': 'daemons', 'disk-writer': 'disk-io', files: 'files', tcp: 'tcp', disk: 'disk', service: 'services', supercode: 'supercode', tabs: 'browser-tabs', power: 'power' };
 // The families a feature's read observes: when its read fails, each is unread (judged neither raised nor cleared).
 // `processes` also observes the renderer, daemon and disk-writer families it reads beside its own; `disk-io`'s own read
 // (device throughput) judges no family: the disk writers are the process read's, so a throughput read that fails leaves
@@ -56,11 +63,18 @@ export class Sampler {
     this.serviceRuns = new Map(); // service label → spawn count at the last reading
     this.recentListeners = new Map(); // port → when it was last seen listening (its TIME_WAITs outlive it)
     this.topFoot = { at: 0, map: new Map() };
+    this.fse = null;             // macOS: fseventsd's event-table reading (#fsevents), kept between its reads
+    this.fseDue = 0;
+    this.fseRunning = null;
+    this.stopping = new AbortController(); // ends the children of the reads that run beside the loop (stop())
     this.slow = { at: 0, readings: {}, observations: [], families: new Set(), errors: {} };
     this.slowStarted = 0;
     this.slowRunning = null;
   }
 
+  /** The probe is stopping: the fseventsd read's children (`log show`, `top`) end with it, and no read starts. */
+  stop() { this.stopping.abort(); }
+
   /** One cycle. `features` is resolveFeatures's output; the slow set runs when its interval has passed. */
   async sample(features, now = Date.now(), { wait = false } = {}) {
     const on = new Set(features.filter((f) => f.on).map((f) => f.id));
@@ -138,7 +152,8 @@ export class Sampler {
         const same = prev && prev.command === r.command && r.cpuSec >= prev.cpuSec;
         r.cpuPct = same && dt ? ((r.cpuSec - prev.cpuSec) / dt) * 100 : null;
         const ru = rus.get(r.pid);
-        const top = this.topFoot.map.get(r.pid);
+        // fseventsd's own footprint is read for it alone (#fsevents): top's list holds only the 25 largest
+        const top = (this.fse?.pid === r.pid ? this.fse.footprintBytes : null) ?? this.topFoot.map.get(r.pid);
         r.footprintBytes = ru?.footprintBytes ?? top ?? r.rssBytes;
         r.footprintSource = ru ? 'footprint' : top != null ? 'footprint (top)' : 'rss';
         r.writeBps = same && dt && ru && prev.written != null && ru.diskWrittenBytes >= prev.written ? (ru.diskWrittenBytes - prev.written) / dt : null;
@@ -229,8 +244,14 @@ export class Sampler {
         for (const r of daemons) {
           const name = base(r.command);
           if (r.cpuPct != null) observations.push({ key: `daemon.cpu:${name}:${r.pid}`, family: 'daemon', def: alarm('daemon.cpu'), value: r.cpuPct, unit: '% CPU', summary: `${name} at ${round(r.cpuPct, 0)}% CPU`, detail: view(r) });
-          const line = footDef.perDaemonMB?.[name] ?? footDef.warnMB;
-          if (r.footprintSource !== 'rss' || this.platform !== 'darwin') observations.push({ key: `daemon.footprint:${name}:${r.pid}`, family: 'daemon', def: { warn: line, forSec: 0 }, value: r.footprintBytes / MB, unit: 'MB', summary: `${name} holds ${Math.round(r.footprintBytes / MB)} MB`, detail: view(r) });
+          const own = footDef.perDaemonMB?.[name];
+          const line = own && typeof own === 'object' ? own : { warn: own ?? footDef.warnMB };
+          const mb = r.footprintBytes / MB;
+          // macOS RSS leaves out compressed pages, so it is no footprint: where a daemon's footprint could not be read
+          // it is judged only for a daemon with its own line, and only past that line (an RSS past it is no less)
+          const resident = r.footprintSource === 'rss' && this.platform === 'darwin';
+          if (!resident || (own != null && mb > line.warn)) observations.push({ key: `daemon.footprint:${name}:${r.pid}`, family: 'daemon', def: { warn: line.warn, crit: line.crit, forSec: 0 }, value: mb, unit: 'MB',
+            summary: `${name} holds ${Math.round(mb)} MB${resident ? ' resident (its footprint was not read)' : ''}${mb > line.warn && DAEMON_PAST[name] ? `: ${DAEMON_PAST[name]}` : ''}`, detail: view(r) });
         }
       }
 
@@ -321,6 +342,57 @@ export class Sampler {
       for (const t of timeWaitByTarget) observations.push({ key: `tcp.timeWaitTarget:${t.target}`, family: 'tcp', def, value: t.pct, unit: '% of ports', summary: `${t.count} TIME_WAIT against ${t.target.startsWith(':') ? `listener ${t.target}${t.exited ? ' (listener has exited; its TIME_WAITs are draining)' : ''}` : t.target}${t.listener ? ` (pid ${t.listener.pid})` : ''}; range ${size}, TIME_WAIT lasts ${timeWaitSec} s`, detail: t });
     });
 
+    // ---- macOS: fseventsd's event table ---------------------------------------------------------------
+    // Its own read beside the fast loop (`log show` takes seconds), merged from what it last found.
+    if (on.has('daemons') && this.reader.fseventsOverflow) {
+      const def = alarm('fseventsd.overflow');
+      if (!this.fseRunning && now >= this.fseDue && !this.stopping.signal.aborted) {
+        this.fseDue = now + def.everySec * 1000;
+        this.fseRunning = this.#fsevents(def, now)
+          .then((more) => { if (more) this.fseDue = 0; }, (error) => { this.fse = { ...(this.fse ?? { pid: null }), unlisted: true, unread: error.message }; })
+          .finally(() => { this.fseRunning = null; });
+      }
+      if (wait && this.fseRunning) await this.fseRunning;
+      const fse = this.fse;
+      if (fse) families.add('fsevents');
+      if (fse?.unlisted) observations.push({ unreadFamily: 'fsevents', family: 'fsevents', why: `fseventsd could not be read: ${fse.unread}` });
+      else if (fse) {
+        // the one fseventsd was read: an alarm on another pid is on a daemon that is gone (it was restarted)
+        observations.push({ enumerated: 'fsevents', family: 'fsevents', by: 'pid', entities: new Set(fse.pid ? [fse.pid] : []) });
+        if (fse.pid) {
+          const key = `fseventsd.overflow:${fse.pid}`;
+          const startedAt = new Date(fse.startedAt).toISOString(), logReadTo = new Date(fse.readTo).toISOString();
+          const unread = fse.unread ? `the unified log could not be read for fseventsd's "${FSE_LINE}" line: ${fse.unread}`
+            : fse.behind ? `the unified log is read for fseventsd's "${FSE_LINE}" line from its start (${startedAt}) only as far as ${logReadTo} yet` : null;
+          readings.fseventsd = { pid: fse.pid, startedAt, logReadTo, eventTable: fse.overflow ? 'overflowed' : unread ? 'unknown' : 'not overflowed', overflowAt: fse.overflow?.at ?? null, ...(unread && !fse.overflow ? { unread } : {}) };
+          const row = procs?.byPid.get(fse.pid);
+          const holds = row ? ` (now ${Math.round(row.footprintBytes / MB)} MB${row.footprintSource === 'rss' ? ' resident' : ''}; 3–60 MB is normal)` : ' (3–60 MB is normal)';
+          if (fse.overflow) observations.push({ key, family: 'fsevents', def, value: 1, met: { warn: true, crit: true, clears: false },
+            summary: `fseventsd logged "${FSE_LINE}" at ${fse.overflow.at ?? 'a time that could not be read'}: its path table is at its maximum, and once its cleanup has stopped it frees nothing, so every new file adds to its memory until it is restarted. Look at its footprint${holds}: ${FSE_GROWTH}. ${FSE_ACT}`, detail: { pid: fse.pid, command: 'fseventsd', startedAt, loggedAt: fse.overflow.at, line: fse.overflow.line } });
+          else if (unread) observations.push({ unreadKey: key, family: 'fsevents', why: unread });
+          else observations.push({ key, family: 'fsevents', def, value: 0, met: { warn: false, crit: false, clears: true }, summary: `fseventsd ${fse.pid} has not logged "${FSE_LINE}" since it started at ${startedAt}` });
+        }
+      }
+      // The guard is not the pack's: the pack reads the files it leaves. Where it is not installed nothing on this Mac
+      // performs the restart; that is a standing state, said when it begins and when it ends and shown in every status.
+      try {
+        const guard = this.reader.fseventsdGuard(alarm('daemon.footprint').perDaemonMB?.fseventsd?.crit);
+        const ranAt = Date.parse(guard.lastRunAt ?? guard.installedAt ?? '');
+        const stale = guard.installed && !(now - ranAt < def.guardSilentMin * 60_000);
+        const line = guard.install ? `\`${guard.install}\`` : 'with the installer in a supercode release (this install carries none: guard/fseventsd-guard-install.sh)';
+        const blind = guard.installed && !stale && guard.unreadRuns >= def.guardUnreadRuns;
+        const state = !guard.installed ? `not installed: nothing on this Mac restarts fseventsd when it has logged "${FSE_LINE}" and its memory grows. Installing it is the one act on this machine no agent can take (none has root); a person runs it once: ${line}`
+          : stale ? `installed, but it has not run since ${guard.lastRunAt ?? guard.installedAt} (it runs every 5 min). A person installs it again: ${line}`
+          : blind ? `installed, but it could not read the unified log on its last ${guard.unreadRuns} runs (${guard.lastRun}): a guard that cannot read is not guarding`
+          : `installed, last run ${guard.lastRunAt ?? 'not yet'}${guard.lastRun ? ` (${guard.lastRun})` : ''}`;
+        const [actAt, act, ...actRest] = (guard.lastAct ?? '').split(' ');
+        const acted = guard.lastAct ? `fseventsd guard ${act === 'restarted' ? 'restarted' : 'could not restart'} the daemon at ${actAt} (${actRest.join(' ')})` : null;
+        readings.fseventsdGuard = { ...guard, state: `fseventsd guard: ${state}`, acted };
+        families.add('fsevents-guard');
+        observations.push({ key: 'fseventsd.guard', family: 'fsevents-guard', def: { forSec: 0 }, value: guard.installed && !stale && !blind ? 0 : 1, met: { warn: !guard.installed || stale || blind, crit: false, clears: guard.installed && !stale && !blind }, standing: true, summary: `fseventsd guard: ${state}` });
+      } catch (error) { observations.push({ unreadKey: 'fseventsd.guard', family: 'fsevents-guard', why: `the fseventsd guard's files could not be read: ${error.message}` }); }
+    }
+
     // ---- the slow set ---------------------------------------------------------------------------------
     // It runs beside the fast loop (the connector can take 20 s to answer) and is merged when it lands;
     // `wait` (a single look) waits for it.
@@ -344,6 +416,42 @@ export class Sampler {
     return { readings, observations, families, errors };
   }
 
+  /**
+   * macOS: whether the fseventsd launchd runs has logged that its event table overflowed, and its footprint (top, for
+   * that pid alone: a read that fails leaves the process list's reading). Each read covers the log since
+   * the last one; a daemon not read before is read from its start, a chunk a cycle (`true`: more is left), and a
+   * window that cannot be read in its budget is halved. The line, once read, stands for that pid and the log is not
+   * read for it again. What was read is kept in fseventsd.json, so a probe that restarts takes up where it stopped
+   * and keeps a line the log has since dropped. Anything not read is said (`unread`), never taken as no line.
+   */
+  async #fsevents(def, now) {
+    const file = join(healthDir(this.env), 'fseventsd.json');
+    let fse = this.fse;
+    if (!fse) try { fse = JSON.parse(readFileSync(file, 'utf8')); } catch { /* not read on this machine before */ }
+    let daemon;
+    const signal = this.stopping.signal;
+    try { daemon = await this.reader.fseventsd({ signal }); } catch (error) { this.fse = { ...(fse ?? { pid: null }), unlisted: true, unread: error.message }; return false; }
+    if (!daemon) { this.fse = { pid: null }; return false; }
+    // the same daemon keeps its pid and its start (ps gives the start to the second)
+    if (fse?.pid !== daemon.pid || Math.abs(fse.startedAt - daemon.startedAt) > 60_000) fse = { pid: daemon.pid, startedAt: daemon.startedAt, readTo: daemon.startedAt - 5000, overflow: null };
+    fse = { ...fse, unlisted: false, unread: null, footprintBytes: await this.reader.footprintOf(daemon.pid, { signal }).catch(() => null) };
+    if (fse.overflow) { this.fse = fse; return false; }
+    const chunk = fse.chunkMs ?? def.chunkHours * 3_600_000;
+    const behind = now - fse.readTo > chunk;
+    const to = behind ? fse.readTo + chunk : now;
+    try {
+      const hits = await this.reader.fseventsOverflow({ pid: fse.pid, ...(behind ? { from: fse.readTo, to } : { lastMin: Math.ceil((now - fse.readTo) / 60_000) + 1 }), timeoutMs: def.timeoutSec * 1000, signal });
+      // every record read is this daemon's own: the log was asked by its pid and its image (darwin.mjs overflowPredicate)
+      fse = { ...fse, readTo: to, behind, overflow: hits[0] ?? null };
+    } catch (error) {
+      fse = { ...fse, unread: error.message, ...(/timed out/.test(error.message) && chunk > 300_000 ? { chunkMs: Math.max(300_000, chunk / 2) } : {}) };
+    }
+    this.fse = fse;
+    // kept in memory when it cannot be written: the reading stands either way
+    try { const temp = `${file}.${process.pid}.tmp`; writeFileSync(temp, JSON.stringify(fse, null, 2) + '\n', { mode: 0o600 }); renameSync(temp, file); } catch { /* read again from the daemon's start after a restart */ }
+    return behind && !fse.unread && !fse.overflow;
+  }
+
   async #slow(on, slow, errors, now) {
     const cfg = this.config;
     const alarm = (name) => cfg.alarms[name];

package/lib/thresholds.mjs

@@ -27,7 +27,14 @@ export const DEFAULTS = {
     'process.growth': { minMB: 4096, mbPerMin: 1024 },
     'browser.orphan': { forSec: 600 },
     'daemon.cpu': { warn: 40, crit: 100, clear: 20, forSec: 120 },
-    'daemon.footprint': { warnMB: 4096, perDaemonMB: { fseventsd: 1024 } },
+    // a daemon's own line is a number (its warn) or { warn, crit }
+    'daemon.footprint': { warnMB: 4096, perDaemonMB: { fseventsd: { warn: 192, crit: 256 } } },
+    // macOS: fseventsd's "event table has grown too large" line, read from the unified log each everySec, the window
+    // since the last read, with the daemon's own footprint; a daemon not read before is read from its start,
+    // chunkHours of log a cycle
+    // guardSilentMin: the root guard a person installs (packaging/npm/guard, every 5 min) reads as not running once silent this long,
+    // and as not guarding once guardUnreadRuns of its runs in a row could not read the log
+    'fseventsd.overflow': { forSec: 0, everySec: 120, chunkHours: 6, timeoutSec: 30, guardSilentMin: 15, guardUnreadRuns: 3 },
     'disk.free': { below: true, warnPct: 10, critPct: 5, forSec: 1800 },
     'disk.filling': { warnHours: 24, warnBelowPct: 40, critHours: 4, lookbackMin: 60 },
     'disk.writer': { warn: 50, forSec: 300 },
@@ -89,6 +96,8 @@ const RANK = { none: 0, warn: 1, crit: 2 };
  * - its feature is off (`{ off: family }`).
  * Anything else not observed is unknown: the alarm holds, saying since when and why (`{ unreadFamily | unreadKey |
  * unreadPrefix, why }` give the why where a site knows it; otherwise it is "not observed this cycle").
+ * An observation marked `standing` is a known state, not an event: its record carries the mark, and it is mailed when
+ * it raises and when it clears, never as a reminder (probe.mjs).
  * `observations` also carry the readings: [{ key, family, def, value, met?, unit, summary, detail }].
  * Every unknown (raised or not: a key or a family that could not be read) is kept in `unknowns` (key → { why,
  * since }) for its notification (probe.mjs); one read again leaves it.
@@ -139,7 +148,7 @@ export function judge(state, observations, families, now, unknowns = {}) {
     else if (m.warn && now - s.warnSince >= warnFor) target = 'warn';
     // Hysteresis: a raised alarm holds at warn until the value is past its clear line.
     if (target === 'none' && s.level !== 'none' && !m.clears) target = 'warn';
-    Object.assign(s, { value: o.value, unit: o.unit ?? null, summary: o.summary, detail: o.detail ?? null, at: now, unread: null, unreadAt: null });
+    Object.assign(s, { value: o.value, unit: o.unit ?? null, summary: o.summary, detail: o.detail ?? null, at: now, unread: null, unreadAt: null, standing: Boolean(o.standing) });
     if (target !== s.level) {
       transitions.push({ key: o.key, from: s.level, to: target, value: o.value, unit: o.unit ?? null, summary: o.summary, detail: o.detail ?? null });
       if (RANK[target] > RANK[s.level] && s.level === 'none') s.raisedAt = now;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@volter/supercode-health",
-  "version": "0.1.21",
+  "version": "0.1.22",
   "type": "module",
   "description": "Optional machine pack: a cheap, deterministic probe of a machine's vitals that mails the machine's maintainer when a threshold is crossed. Each feature sits behind its own permission barrier.",
   "license": "MIT",