@volter/supercode-health 0.0.0-stage → 0.1.0

package/lib/sample.mjs

@@ -0,0 +1,492 @@
+// The sampler: reads every feature that is on, turns cumulative counters into rates against the previous
+// sample, and states each alarm's observation. It keeps only what a rate or a window needs between samples.
+import { cpus, homedir, loadavg } from 'node:os';
+import { existsSync, readFileSync, statSync } from 'node:fs';
+import { run } from './run.mjs';
+import { rendererOrigins, originOf, localPort } from './chrome.mjs';
+
+const MB = 1024 * 1024;
+const GB = 1024 * MB;
+const round = (n, d = 1) => (n == null || Number.isNaN(n) ? null : Math.round(n * 10 ** d) / 10 ** d);
+const short = (command) => String(command ?? '').slice(0, 160);
+const base = (command) => {
+  const first = String(command ?? '').split(' ')[0];
+  return first.slice(first.lastIndexOf('/') + 1).replace(/^\[|\]$/g, '');
+};
+const AGENT = /^(claude|codex|gemini|goose|opencode|grok|hermes|pi|aider|cursor-agent)$/;
+const HARNESS = { claude: 'claude-code', codex: 'codex', gemini: 'gemini', goose: 'goose', opencode: 'opencode', grok: 'grok', hermes: 'hermes', pi: 'pi' };
+const RENDERER = /Helper \(Renderer\)|--type=renderer/;
+const BROWSER_MAIN = /(Google Chrome|Chromium|chrome-headless-shell|chrome)(\s|$)/i;
+const LOOPBACK = /^(127\.|::1$|localhost$)/;
+// The slow set's readings and alarm families, by the feature that produces them.
+const SLOW_FEATURE = { disk: 'disk', services: 'services', service: 'services', supercode: 'supercode', browserTabs: 'browser-tabs', tabs: 'browser-tabs', power: 'power' };
+// Every alarm family, by the feature that observes it.
+const FAMILY_FEATURE = { load: 'load', memory: 'memory', process: 'processes', browser: 'renderers', daemon: 'daemons', 'disk-writer': 'disk-io', files: 'files', tcp: 'tcp', disk: 'disk', service: 'services', supercode: 'supercode', tabs: 'browser-tabs', power: 'power' };
+
+export class Sampler {
+  constructor({ reader, platform = process.platform, machine, env = process.env, config }) {
+    Object.assign(this, { reader, platform, machine, env, config });
+    this.prev = null;           // the previous fast sample's counters
+    this.procPrev = new Map();  // pid → { cpuSec, command, written }
+    this.footHistory = new Map(); // pid → [[at, bytes]] for growth per minute
+    this.swapRing = [];          // [[at, swapouts]] for the 30-minute sum
+    this.diskRing = new Map();   // mount → [[at, freeBytes]] for the fill rate
+    this.sessionCache = new Map(); // pid:command → session
+    this.topFoot = { at: 0, map: new Map() };
+    this.slow = { at: 0, readings: {}, observations: [], families: new Set(), errors: {} };
+    this.slowStarted = 0;
+    this.slowRunning = null;
+  }
+
+  /** One cycle. `features` is resolveFeatures's output; the slow set runs when its interval has passed. */
+  async sample(features, now = Date.now(), { wait = false } = {}) {
+    const on = new Set(features.filter((f) => f.on).map((f) => f.id));
+    const cfg = this.config;
+    this.procs = null;
+    const readings = {};
+    const observations = [];
+    const families = new Set();
+    const errors = {};
+    const dt = this.prev ? (now - this.prev.at) / 1000 : null;
+    const attempt = async (id, fn) => {
+      try { await fn(); } catch (error) { errors[id] = error.message; }
+    };
+    const alarm = (name) => cfg.alarms[name];
+
+    if (on.has('load')) await attempt('load', async () => {
+      const [one, five, fifteen] = loadavg();
+      const cores = cpus().length;
+      readings.load = { one: round(one, 2), five: round(five, 2), fifteen: round(fifteen, 2), cores, perCore: round(one / cores, 2) };
+      families.add('load');
+      if (this.platform !== 'win32') observations.push({ key: 'load.perCore', family: 'load', def: alarm('load.perCore'), value: one / cores, unit: '× cores', summary: `load ${round(one, 1)} on ${cores} cores` });
+    });
+
+    let memory = null;
+    if (on.has('memory')) await attempt('memory', async () => {
+      memory = await this.reader.memory();
+      const c = memory.counters;
+      const p = this.prev?.memory?.counters;
+      const rate = (k) => (p && dt && c[k] != null && p[k] != null && c[k] >= p[k] ? (c[k] - p[k]) / dt : null);
+      if (c.swapouts != null) {
+        this.swapRing.push([now, c.swapouts]);
+        while (this.swapRing.length && now - this.swapRing[0][0] > 31 * 60_000) this.swapRing.shift();
+      }
+      const oldest = this.swapRing[0];
+      const swapped30 = oldest && c.swapouts != null && now - oldest[0] >= 60_000 ? ((c.swapouts - oldest[1]) * memory.pageSize * 100) / memory.totalBytes : null;
+      readings.memory = {
+        pressureLevel: memory.pressureLevel, freePct: memory.freePct, totalGB: round(memory.totalBytes / GB),
+        swapoutsPerSec: round(rate('swapouts')), swapinsPerSec: round(rate('swapins')), pageinsPerSec: round(rate('pageins')),
+        compressionsPerSec: round(rate('compressions'), 0), swapUsedGB: round(memory.swapUsedBytes / GB, 2), swapTotalGB: round(memory.swapTotalBytes / GB, 2),
+        swappedPctRamWindow: round(swapped30, 2), swappedWindowMin: oldest ? round((now - oldest[0]) / 60_000, 0) : null,
+        ...(memory.psi ? { psi: memory.psi } : {}),
+      };
+      families.add('memory');
+      if (memory.pressureLevel) observations.push({ key: 'memory.pressure', family: 'memory', def: alarm('memory.pressure'), value: memory.pressureLevel, met: { warn: memory.pressureLevel !== 'normal', crit: memory.pressureLevel === 'critical', clears: memory.pressureLevel === 'normal' }, summary: `kernel memory pressure ${memory.pressureLevel}` });
+      if (memory.freePct != null) observations.push({ key: 'memory.free', family: 'memory', def: alarm('memory.free'), value: memory.freePct, unit: '%', summary: `memory free ${memory.freePct}%` });
+      if (rate('swapouts') != null) observations.push({ key: 'memory.swapping', family: 'memory', def: alarm('memory.swapping'), value: rate('swapouts'), unit: 'pages/s', summary: `swapping out ${round(rate('swapouts'))} pages/s` });
+      if (swapped30 != null) observations.push({ key: 'memory.swapped30m', family: 'memory', def: alarm('memory.swapped30m'), value: swapped30, unit: '% of RAM', summary: `${round(swapped30, 2)}% of RAM swapped out in the last ${readings.memory.swappedWindowMin} min` });
+      if (rate('compressions') != null) observations.push({ key: 'memory.compressions', family: 'memory', def: alarm('memory.compressions'), value: rate('compressions'), unit: 'pages/s', summary: `${round(rate('compressions'), 0)} compressions/s` });
+    });
+
+    // ---- processes, renderers, daemons, writers ------------------------------------------------------
+    let procs = null;
+    if (on.has('processes')) await attempt('processes', async () => {
+      const [rows, rus] = await Promise.all([
+        this.reader.processes(),
+        on.has('process-footprint') ? this.reader.rusage().catch((error) => { errors['process-footprint'] = error.message; return new Map(); }) : new Map(),
+      ]);
+      if (this.platform === 'darwin' && now - this.topFoot.at >= cfg.footprintTopEverySec * 1000) {
+        this.topFoot = { at: now, map: await this.reader.topFootprints().catch((error) => { errors.daemons = error.message; return this.topFoot.map; }) };
+      }
+      const byPid = new Map(rows.map((r) => [r.pid, r]));
+      const nextPrev = new Map();
+      for (const r of rows) {
+        const prev = this.procPrev.get(r.pid);
+        const same = prev && prev.command === r.command && r.cpuSec >= prev.cpuSec;
+        r.cpuPct = same && dt ? ((r.cpuSec - prev.cpuSec) / dt) * 100 : null;
+        const ru = rus.get(r.pid);
+        const top = this.topFoot.map.get(r.pid);
+        r.footprintBytes = ru?.footprintBytes ?? top ?? r.rssBytes;
+        r.footprintSource = ru ? 'footprint' : top != null ? 'footprint (top)' : 'rss';
+        r.writeBps = same && dt && ru && prev.written != null && ru.diskWrittenBytes >= prev.written ? (ru.diskWrittenBytes - prev.written) / dt : null;
+        nextPrev.set(r.pid, { cpuSec: r.cpuSec, command: r.command, written: ru?.diskWrittenBytes ?? null });
+        const hist = same ? (this.footHistory.get(r.pid) ?? []) : [];
+        hist.push([now, r.footprintBytes]);
+        while (hist.length && now - hist[0][0] > 130_000) hist.shift();
+        this.footHistory.set(r.pid, hist);
+        const ago = hist.findLast?.(([at]) => now - at >= 55_000) ?? [...hist].reverse().find(([at]) => now - at >= 55_000);
+        r.growthMBPerMin = ago ? ((r.footprintBytes - ago[1]) / MB) * (60_000 / (now - ago[0])) : null;
+      }
+      for (const pid of this.footHistory.keys()) if (!nextPrev.has(pid)) this.footHistory.delete(pid);
+      this.procPrev = nextPrev;
+      procs = { rows, byPid };
+    });
+
+    if (procs) {
+      const { rows, byPid } = procs;
+      const self = process.pid;
+      const listed = new Set();
+      const view = (r) => { listed.add(r.pid); return { pid: r.pid, cpuPct: round(r.cpuPct, 0), footprintMB: Math.round(r.footprintBytes / MB), footprintSource: r.footprintSource, growthMBPerMin: round(r.growthMBPerMin, 0), command: short(r.command) }; };
+      const byCpu = rows.filter((r) => r.cpuPct != null && r.pid !== self).sort((a, b) => b.cpuPct - a.cpuPct);
+      const byMem = [...rows].sort((a, b) => b.footprintBytes - a.footprintBytes);
+      const topCpu = byCpu.slice(0, 10).map(view);
+      const topMemory = byMem.slice(0, 10).map(view);
+      readings.processes = { count: rows.length, topCpu, topMemory };
+      families.add('process');
+      const daemonNames = new Set(cfg.daemons?.[this.platform] ?? []);
+      const isDaemon = (r) => daemonNames.has(base(r.command));
+      // Every process down to the lower clear line is observed, so a raised alarm holds through a brief dip
+      // instead of clearing and raising again (hysteresis); one that falls below it is no longer watched.
+      for (const r of byCpu) {
+        if (r.cpuPct < Math.min(alarm('process.cpu').clear ?? alarm('process.cpu').warn, alarm('renderer.cpu').clear ?? alarm('renderer.cpu').warn)) break;
+        if (isDaemon(r) || AGENT.test(base(r.command))) continue;
+        const renderer = RENDERER.test(r.command);
+        const name = renderer ? 'renderer.cpu' : 'process.cpu';
+        const tab = renderer ? this.rendererTabs?.get(r.pid) ?? null : null;
+        observations.push({ key: `${name}:${r.pid}`, family: 'process', def: alarm(name), value: r.cpuPct, unit: '% CPU', summary: `${renderer ? 'renderer' : 'process'} ${r.pid} at ${round(r.cpuPct, 0)}% CPU${tab ? `, serving ${tab.url.slice(0, 80)} (${tab.owner})` : ''}`, detail: { ...view(r), ...(renderer ? { tab } : {}) } });
+      }
+      // Runaway growth: one reading against the reading a minute earlier. Every process over the size line is
+      // observed; a raised one holds until its growth falls under a quarter of the line, so a leak that pauses
+      // between samples is one alarm, not a raise and a clear each minute.
+      for (const r of rows) {
+        if (r.growthMBPerMin == null) continue;
+        const renderer = RENDERER.test(r.command);
+        const tab = renderer ? this.rendererTabs?.get(r.pid) ?? null : null;
+        const def = alarm(renderer ? 'renderer.growth' : 'process.growth');
+        if (r.footprintBytes / MB < def.minMB) continue;
+        observations.push({ key: `${renderer ? 'renderer' : 'process'}.growth:${r.pid}`, family: 'process', def: { forSec: 0 }, value: r.growthMBPerMin, met: { warn: r.growthMBPerMin > def.mbPerMin, crit: false, clears: r.growthMBPerMin < def.mbPerMin / 4 }, unit: 'MB/min', summary: `${renderer ? 'renderer' : 'process'} ${r.pid} at ${Math.round(r.footprintBytes / MB)} MB, growing ${Math.round(r.growthMBPerMin)} MB/min${tab ? `, serving ${tab.url.slice(0, 80)} (${tab.owner})` : ''}`, detail: { ...view(r), ...(renderer ? { tab } : {}) } });
+      }
+
+      if (on.has('renderers')) {
+        const renderers = rows.filter((r) => RENDERER.test(r.command));
+        const orphaned = rows.filter((r) => r.ppid === 1 && /--headless/.test(r.command) && !/--type=/.test(r.command) && BROWSER_MAIN.test(r.command));
+        this.renderers = renderers.map((r) => ({ pid: r.pid, footprintBytes: r.footprintBytes, cpuPct: r.cpuPct }));
+        // The tab each renderer serves, as the browser-tabs feature last matched it (null: not matched).
+        const tabbed = (v) => ({ ...v, tab: this.rendererTabs?.get(v.pid) ?? null });
+        readings.renderers = {
+          count: renderers.length,
+          totalMB: Math.round(renderers.reduce((n, r) => n + r.footprintBytes, 0) / MB),
+          top: renderers.sort((a, b) => (b.cpuPct ?? 0) - (a.cpuPct ?? 0) || b.footprintBytes - a.footprintBytes).slice(0, 5).map(view).map(tabbed),
+          orphanedHeadless: orphaned.map(view),
+        };
+        families.add('browser');
+        for (const r of orphaned) observations.push({ key: `browser.orphan:${r.pid}`, family: 'browser', def: alarm('browser.orphan'), value: 1, met: { warn: true, crit: false, clears: false }, summary: `headless browser ${r.pid} left running with launchd as its parent`, detail: view(r) });
+      }
+
+      if (on.has('daemons')) {
+        const daemons = rows.filter(isDaemon);
+        readings.daemons = daemons.sort((a, b) => (b.cpuPct ?? 0) - (a.cpuPct ?? 0)).map((r) => ({ name: base(r.command), ...view(r) }));
+        families.add('daemon');
+        const footDef = alarm('daemon.footprint');
+        for (const r of daemons) {
+          const name = base(r.command);
+          if (r.cpuPct != null) observations.push({ key: `daemon.cpu:${name}:${r.pid}`, family: 'daemon', def: alarm('daemon.cpu'), value: r.cpuPct, unit: '% CPU', summary: `${name} at ${round(r.cpuPct, 0)}% CPU`, detail: view(r) });
+          const line = footDef.perDaemonMB?.[name] ?? footDef.warnMB;
+          if (r.footprintSource !== 'rss' || this.platform !== 'darwin') observations.push({ key: `daemon.footprint:${name}:${r.pid}`, family: 'daemon', def: { warn: line, forSec: 0 }, value: r.footprintBytes / MB, unit: 'MB', summary: `${name} holds ${Math.round(r.footprintBytes / MB)} MB`, detail: view(r) });
+        }
+      }
+
+      if (on.has('disk-io')) {
+        const writers = rows.filter((r) => r.writeBps > 0).sort((a, b) => b.writeBps - a.writeBps).slice(0, 5);
+        readings.diskWriters = writers.map((r) => ({ ...view(r), writeMBps: round(r.writeBps / MB, 1) }));
+        families.add('disk-writer');
+        for (const r of rows) if (r.writeBps != null && r.writeBps / MB > alarm('disk.writer').warn * 0.5) observations.push({ key: `disk.writer:${r.pid}`, family: 'disk-writer', def: alarm('disk.writer'), value: r.writeBps / MB, unit: 'MB/s', summary: `process ${r.pid} writing ${round(r.writeBps / MB, 1)} MB/s`, detail: view(r) });
+      }
+
+      // Who each listed process belongs to: the agent session above it, read from the inherited environment.
+      this.procs = procs;
+      this.listed = listed;
+    }
+
+    if (on.has('disk-io')) await attempt('disk-io', async () => {
+      const io = await this.reader.diskIO();
+      const p = this.prev?.io;
+      readings.diskIO = Object.fromEntries(Object.entries(io).map(([name, v]) => [name, { MBps: p?.[name] && dt ? round((v.bytes - p[name].bytes) / dt / MB, 1) : null }]));
+      this.io = io;
+    });
+
+    if (on.has('files')) await attempt('files', async () => {
+      const { open, max } = await this.reader.files();
+      readings.files = { open, max, pct: round((open / max) * 100) };
+      families.add('files');
+      observations.push({ key: 'files.open', family: 'files', def: alarm('files.open'), value: (open / max) * 100, unit: '%', summary: `${open} of ${max} files open` });
+    });
+
+    if (on.has('tcp')) await attempt('tcp', async () => {
+      const { sockets, range, timeWaitSec } = await this.reader.tcp();
+      this.sockets = sockets;
+      const size = range.last - range.first + 1;
+      const states = {};
+      const listening = new Set();
+      const local = new Set();
+      for (const s of sockets) {
+        states[s.state] = (states[s.state] ?? 0) + 1;
+        if (s.state === 'LISTEN') listening.add(s.local.port);
+        if (!['*', '0.0.0.0', '::', '[::]'].includes(s.local.addr)) local.add(s.local.addr);
+      }
+      const inRange = (port) => port >= range.first && port <= range.last;
+      const wildcard = new Set();
+      const used = new Map();
+      const add = (addr, port) => { if (!used.has(addr)) used.set(addr, new Set()); used.get(addr).add(port); };
+      const targets = new Map();
+      for (const s of sockets) {
+        if (inRange(s.local.port)) {
+          if (['*', '0.0.0.0', '::', '[::]'].includes(s.local.addr)) wildcard.add(s.local.port); else add(s.local.addr, s.local.port);
+        }
+        if (local.has(s.foreign.addr) && inRange(s.foreign.port)) add(s.foreign.addr, s.foreign.port);
+        if (s.state !== 'TIME_WAIT') continue;
+        // Group each TIME_WAIT by the endpoint it churned against: a listener here (either side), else the remote.
+        let target;
+        if (listening.has(s.local.port)) target = `:${s.local.port}`;
+        else if ((local.has(s.foreign.addr) || LOOPBACK.test(s.foreign.addr)) && listening.has(s.foreign.port)) target = `:${s.foreign.port}`;
+        else target = `${s.foreign.addr}:${s.foreign.port}`;
+        targets.set(target, (targets.get(target) ?? 0) + 1);
+      }
+      const ephemeral = [...used].map(([addr, ports]) => {
+        for (const port of wildcard) ports.add(port);
+        return { addr, used: ports.size, pct: round((ports.size / size) * 100) };
+      }).sort((a, b) => b.used - a.used).slice(0, 5);
+      const top = [...targets].sort((a, b) => b[1] - a[1]).slice(0, 5);
+      let owners = new Map();
+      const def = alarm('tcp.timeWaitTarget');
+      if (top.some(([target, n]) => target.startsWith(':') && (n / size) * 100 >= def.warn / 2) || now - (this.listenersAt ?? 0) >= cfg.slowIntervalSec * 1000) {
+        owners = this.reader.listeners ? await this.reader.listeners(sockets).catch(() => new Map()) : new Map();
+        this.owners = owners; this.listenersAt = now;
+      } else owners = this.owners ?? new Map();
+      const timeWaitByTarget = top.map(([target, count]) => {
+        const owner = target.startsWith(':') ? owners.get(Number(target.slice(1))) : null;
+        return { target, count, pct: round((count / size) * 100), ...(owner ? { listener: { pid: owner.pid, command: short(owner.command) } } : {}) };
+      });
+      readings.tcp = { states, timeWait: states.TIME_WAIT ?? 0, range: { ...range, size }, timeWaitSec, ephemeral, timeWaitByTarget };
+      families.add('tcp');
+      for (const e of ephemeral) observations.push({ key: `tcp.ephemeral:${e.addr}`, family: 'tcp', def: alarm('tcp.ephemeral'), value: e.pct, unit: '% of ports', summary: `${e.used} of ${size} ephemeral ports in use on ${e.addr}` });
+      for (const t of timeWaitByTarget) observations.push({ key: `tcp.timeWaitTarget:${t.target}`, family: 'tcp', def, value: t.pct, unit: '% of ports', summary: `${t.count} TIME_WAIT against ${t.target.startsWith(':') ? `listener ${t.target}` : t.target}${t.listener ? ` (pid ${t.listener.pid})` : ''}; range ${size}, TIME_WAIT lasts ${timeWaitSec} s`, detail: t });
+    });
+
+    // ---- the slow set ---------------------------------------------------------------------------------
+    // It runs beside the fast loop (the connector can take 20 s to answer) and is merged when it lands;
+    // `wait` (a single look) waits for it.
+    if (!this.slowRunning && now - this.slowStarted >= cfg.slowIntervalSec * 1000) {
+      this.slowStarted = now;
+      const slow = { at: now, readings: {}, observations: [], families: new Set(), errors: {} };
+      this.slowRunning = this.#slow(on, slow, slow.errors, now).then(() => { this.slow = slow; }).finally(() => { this.slowRunning = null; });
+    }
+    if (wait && this.slowRunning) await this.slowRunning;
+    // A slow reading lands up to a minute after it was taken: one whose feature has since been turned off is dropped.
+    for (const [key, value] of Object.entries(this.slow.readings)) if (on.has(SLOW_FEATURE[key])) readings[key] = value;
+    for (const [id, error] of Object.entries(this.slow.errors ?? {})) if (on.has(id)) errors[id] = error;
+    for (const o of this.slow.observations) if (on.has(SLOW_FEATURE[o.family])) observations.push(o);
+    for (const f of this.slow.families ?? []) if (on.has(SLOW_FEATURE[f])) families.add(f);
+
+    // A feature turned off clears its raised alarms: its family is judged with nothing observed.
+    for (const [family, feature] of Object.entries(FAMILY_FEATURE)) if (!on.has(feature)) families.add(family);
+
+    await this.#sessions(readings, observations);
+    this.prev = { at: now, memory, io: this.io ?? null };
+    return { readings, observations, families, errors };
+  }
+
+  async #slow(on, slow, errors, now) {
+    const cfg = this.config;
+    const alarm = (name) => cfg.alarms[name];
+    const attempt = async (id, fn) => { try { await fn(); } catch (error) { errors[id] = error.message; } };
+
+    if (on.has('disk')) await attempt('disk', async () => {
+      const def = alarm('disk.free');
+      const fill = alarm('disk.filling');
+      const vols = await this.reader.volumes();
+      slow.families.add('disk');
+      slow.readings.disk = vols.map((v) => {
+        const ring = this.diskRing.get(v.mount) ?? [];
+        ring.push([now, v.freeBytes]);
+        while (ring.length && now - ring[0][0] > (fill.lookbackMin + 1) * 60_000) ring.shift();
+        this.diskRing.set(v.mount, ring);
+        const [t0, f0] = ring[0];
+        const hours = (now - t0) / 3_600_000;
+        const perHour = hours >= 10 / 60 ? (f0 - v.freeBytes) / hours : null; // bytes consumed per hour
+        const hoursToFull = perHour > 0 ? v.freeBytes / perHour : null;
+        const freePct = (v.freeBytes / v.totalBytes) * 100;
+        const per = cfg.volumes?.[v.mount] ?? {};
+        const row = { mount: v.mount, freeGB: round(v.freeBytes / GB), totalGB: round(v.totalBytes / GB), freePct: round(freePct), fillGBPerHour: perHour == null ? null : round(perHour / GB, 2), hoursToFull: round(hoursToFull) };
+        const byGB = per.warnGB != null || per.critGB != null;
+        slow.observations.push(byGB
+          ? { key: `disk.free:${v.mount}`, family: 'disk', def: { below: true, warn: per.warnGB, crit: per.critGB, clear: per.clearGB, forSec: per.forSec ?? def.forSec }, value: v.freeBytes / GB, unit: 'GB', summary: `${v.mount} has ${round(v.freeBytes / GB)} GB free` }
+          : { key: `disk.free:${v.mount}`, family: 'disk', def: { below: true, warn: per.warnPct ?? def.warnPct, crit: per.critPct ?? def.critPct, forSec: per.forSec ?? def.forSec }, value: freePct, unit: '%', summary: `${v.mount} has ${round(freePct)}% (${round(v.freeBytes / GB)} GB) free` });
+        if (hoursToFull != null) slow.observations.push({ key: `disk.filling:${v.mount}`, family: 'disk', def: { forSec: 0 }, value: hoursToFull, unit: 'h to full', met: { crit: hoursToFull < fill.critHours, warn: hoursToFull < fill.critHours || (hoursToFull < fill.warnHours && freePct < fill.warnBelowPct), clears: hoursToFull >= fill.warnHours || freePct >= fill.warnBelowPct }, summary: `${v.mount} fills in ${round(hoursToFull)} h at ${round(perHour / GB, 2)} GB/h` });
+        return row;
+      });
+    });
+
+    if (on.has('services')) await attempt('services', async () => {
+      const list = cfg.services ?? [];
+      const jobs = await this.reader.services(list.map((s) => s.label));
+      slow.families.add('service');
+      slow.readings.services = list.map((s) => {
+        const job = jobs.get(s.label);
+        let logAgeMin = null;
+        const log = s.log?.replace(/^~(?=\/)/, homedir());
+        if (log) try { logAgeMin = Math.max(0, round((now - statSync(log).mtimeMs) / 60_000)); } catch { logAgeMin = null; }
+        const running = Boolean(job?.pid);
+        const down = !job || (!running && (job.lastExit !== 0 || s.keepAlive !== false));
+        const why = !job ? 'not loaded' : !running ? `not running (last exit ${job.lastExit})` : null;
+        if (down) slow.observations.push({ key: `service.down:${s.label}`, family: 'service', def: alarm('service.down'), value: 1, met: { warn: true, crit: false, clears: false }, summary: `service ${s.name ?? s.label} is ${why}` });
+        else slow.observations.push({ key: `service.down:${s.label}`, family: 'service', def: alarm('service.down'), value: 0, met: { warn: false, crit: false, clears: true }, summary: `service ${s.name ?? s.label} running` });
+        if (s.log && s.maxSilentMin) slow.observations.push({ key: `service.deaf:${s.label}`, family: 'service', def: { warn: s.maxSilentMin, forSec: 0 }, value: logAgeMin ?? Infinity, unit: 'min silent', summary: logAgeMin == null ? `service ${s.name ?? s.label}: its log ${log} is missing` : `service ${s.name ?? s.label} has written nothing to its log for ${logAgeMin} min` });
+        return { label: s.label, name: s.name ?? null, running, pid: job?.pid ?? null, lastExit: job?.lastExit ?? null, ...(s.log ? { logAgeMin } : {}) };
+      });
+    });
+
+    if (on.has('supercode')) await attempt('supercode', async () => {
+      const bin = this.env.SUPERCODE_BIN || 'supercode';
+      slow.families.add('supercode');
+      const t0 = Date.now();
+      // The server's live view of this machine comes from the machine list: `teams status` carries the
+      // enrollment record, whose connection field is not kept current.
+      // Liveness is the daemon's cheapest door (`describe`); `launches.list` walks every pane and can be slow
+      // on a busy machine without the daemon being down, so its timeout is a reading, not an alarm.
+      const [status, describe, launches, machines] = await Promise.all([
+        run(bin, ['teams', 'status', '--json'], { timeoutMs: 20_000, env: this.env }),
+        run(bin, ['teams', 'describe'], { timeoutMs: 15_000, env: this.env }).then((r) => ({ ...r, ms: Date.now() - t0 })),
+        run(bin, ['teams', 'launches', 'list'], { timeoutMs: 45_000, env: this.env }),
+        run(bin, ['teams', 'machines', 'list', '--json'], { timeoutMs: 20_000, env: this.env }),
+      ]);
+      let described = null; try { described = JSON.parse(describe.stdout); } catch { /* the daemon did not answer */ }
+      let data = null; try { data = JSON.parse(status.stdout).data; } catch { /* unreadable */ }
+      const connector = data?.connector ?? null;
+      let listed = null;
+      try { const body = JSON.parse(machines.stdout); listed = (body.items ?? body.data?.items ?? []).find((m) => m.id === connector?.machine?.id) ?? null; } catch { /* unreadable */ }
+      const connection = listed?.connection ?? null;
+      let rows = null; try { rows = JSON.parse(launches.stdout); } catch { /* the daemon did not answer */ }
+      const def = alarm('supercode.launch');
+      const recent = Array.isArray(rows) ? rows.filter((r) => r.at && now - r.at < 24 * 3_600_000) : [];
+      const stuck = recent.filter((r) => (r.status === 'issued' && now - r.at > def.stuckMin * 60_000) || (r.live && r.inputDelivered === false && r.inputError));
+      const failed = recent.filter((r) => r.status === 'failed' && now - r.at < def.failedWithinMin * 60_000);
+      slow.readings.supercode = {
+        connector: connector ? { enrolled: connector.enrolled, running: connector.running, connection, notSyncing: connector.sync?.not_syncing?.length ?? 0, lastError: connector.sync?.last_error?.message ?? null } : null,
+        statusError: status.ok ? null : status.error,
+        daemon: { answered: Boolean(described), error: described ? null : (describe.error || 'no answer'), ms: describe.ms },
+        launches: { read: Array.isArray(rows), ...(Array.isArray(rows) ? {} : { error: launches.error || 'no answer' }), recent: recent.length, stuck: stuck.map((r) => ({ key: r.key, pane: r.pane, kind: r.kind, status: r.status, at: new Date(r.at).toISOString(), inputError: r.inputError ?? null })), failed: failed.map((r) => ({ key: r.key, kind: r.kind, at: new Date(r.at).toISOString() })) },
+      };
+      const cdef = alarm('supercode.connector');
+      const link = connector?.enrolled ? (connector.running ? (connection == null ? null : connection === 'online') : false) : true;
+      slow.observations.push({ key: 'supercode.daemon', family: 'supercode', def: cdef, value: described ? 0 : 1, met: { warn: !described, crit: false, clears: Boolean(described) }, summary: described ? 'machine daemon answering' : `machine daemon did not answer describe (${describe.error || 'no answer'})` });
+      if (connector && link !== null) slow.observations.push({ key: 'supercode.link', family: 'supercode', def: cdef, value: link ? 0 : 1, met: { warn: !link, crit: false, clears: link }, summary: link ? 'connector linked to its server' : `connector ${connector.running ? `running but the server sees this machine ${connection}` : 'not running'}` });
+      for (const r of stuck) slow.observations.push({ key: `supercode.launch:${r.key}`, family: 'supercode', def: { forSec: 0 }, value: 1, met: { warn: true, crit: false, clears: false }, summary: r.status === 'issued' ? `launch ${r.key} (${r.kind ?? 'pane'}) issued ${Math.round((now - r.at) / 60_000)} min ago and never became ready` : `launch ${r.key} is live but its first input was not delivered: ${r.inputError}` });
+      for (const r of failed) slow.observations.push({ key: `supercode.launchFailed:${r.key}`, family: 'supercode', def: { forSec: 0 }, value: 1, met: { warn: true, crit: false, clears: false }, summary: `launch ${r.key} (${r.kind ?? 'pane'}) failed at ${new Date(r.at).toISOString()}` });
+    });
+
+    if (on.has('browser-tabs')) await attempt('browser-tabs', async () => {
+      const c = cfg.integrations.browserController;
+      let url = c.url;
+      if (!url && c.portFile) {
+        if (!existsSync(c.portFile)) throw new Error(`browser controller not running (${c.portFile} missing)`);
+        url = `http://127.0.0.1:${readFileSync(c.portFile, 'utf8').trim()}`;
+      }
+      const response = await fetch(`${url}/tabs`, { signal: AbortSignal.timeout(5000) });
+      const body = await response.json();
+      const tabs = body.result ?? body.tabs ?? [];
+      const byHolder = {};
+      for (const t of tabs) for (const h of [...(t.holders ?? []), ...(t.openedFor ? [`openedFor:${t.openedFor}`] : [])]) byHolder[h] = (byHolder[h] ?? 0) + 1;
+      // Who owns each tab: the agent holding it through the controller, else whoever it was opened for, else (a
+      // tab on a local port, such as an editor that opened its own tab) the process serving that port.
+      const listeners = this.reader.listeners ? await this.reader.listeners(this.sockets ?? []).catch(() => new Map()) : new Map();
+      const rows = tabs.map((t) => {
+        const port = localPort(t.url);
+        const server = port ? listeners.get(port) : null;
+        const owner = t.holders?.length ? `held by ${t.holders.join(', ')}` : t.openedFor ? `opened for ${t.openedFor}` : server ? `served by pid ${server.pid} (${server.command})` : port ? `local port ${port} with no listener: a leftover` : 'no owner known';
+        return { title: String(t.title ?? '').slice(0, 80), url: String(t.url ?? '').slice(0, 200), origin: originOf(t.url), holders: t.holders ?? [], openedFor: t.openedFor ?? null, owner, ...(server ? { server: { pid: server.pid, command: short(server.command) } } : {}), renderers: [] };
+      });
+      // Which renderer serves which tab, from the storage buckets each large or busy renderer holds open.
+      const candidates = (this.renderers ?? []).filter((r) => r.footprintBytes >= 300 * MB || (r.cpuPct ?? 0) >= 20).slice(0, 12).map((r) => r.pid);
+      const origins = await rendererOrigins(candidates, this.platform).catch(() => new Map());
+      const rendererTabs = new Map();
+      for (const [pid, set] of origins) for (const row of rows) if (row.origin && set.has(row.origin)) { row.renderers.push(pid); rendererTabs.set(pid, { url: row.url, title: row.title, owner: row.owner, ...(row.server ? { server: row.server } : {}) }); }
+      this.rendererTabs = rendererTabs;
+      slow.readings.browserTabs = { count: tabs.length, byHolder, tabs: rows };
+      slow.families.add('tabs');
+      const def = alarm('browser.tabs');
+      if (def.warn != null) slow.observations.push({ key: 'browser.tabs', family: 'tabs', def, value: tabs.length, unit: 'tabs', summary: `${tabs.length} browser tabs open` });
+    });
+
+    if (on.has('power')) await attempt('power', async () => {
+      const r = await run('/usr/bin/powermetrics', ['-n', '1', '-i', '1000', '--samplers', 'thermal'], { timeoutMs: 10_000 });
+      if (!r.ok) throw new Error(r.error);
+      const level = /pressure level:\s*(\w+)/i.exec(r.stdout)?.[1] ?? null;
+      slow.readings.power = { thermalPressure: level };
+      slow.families.add('power');
+      if (level) slow.observations.push({ key: 'power.thermal', family: 'power', def: alarm('power.thermal'), value: level, met: { warn: level !== 'Nominal', crit: /Heavy|Trapping|Sleeping/.test(level), clears: level === 'Nominal' }, summary: `thermal pressure ${level}` });
+    });
+  }
+
+  /** Names the agent session that owns each process a reading or an alarm names. */
+  async #sessions(readings, observations) {
+    if (!this.procs) return;
+    const { byPid } = this.procs;
+    const wanted = new Set();
+    const views = [];
+    const collect = (v) => { if (v?.pid) { wanted.add(v.pid); views.push(v); } };
+    for (const v of [...(readings.processes?.topCpu ?? []), ...(readings.processes?.topMemory ?? []), ...(readings.renderers?.top ?? []), ...(readings.renderers?.orphanedHeadless ?? []), ...(readings.diskWriters ?? [])]) collect(v);
+    for (const t of readings.tcp?.timeWaitByTarget ?? []) collect(t.listener);
+    for (const t of readings.browserTabs?.tabs ?? []) collect(t.server);
+    for (const r of readings.renderers?.top ?? []) collect(r.tab?.server);
+    for (const o of observations) collect(o.detail?.pid ? o.detail : o.detail?.listener);
+    const chain = (pid) => {
+      const out = []; let p = pid;
+      for (let i = 0; i < 40 && p > 1; i += 1) { const r = byPid.get(p); if (!r) break; out.push(r); p = r.ppid; }
+      return out;
+    };
+    const need = new Set();
+    const plan = new Map();
+    for (const pid of wanted) {
+      const r = byPid.get(pid);
+      if (!r) continue;
+      const cacheKey = `${pid}:${r.command}`;
+      if (this.sessionCache.has(cacheKey)) { plan.set(pid, { cacheKey }); continue; }
+      const up = chain(pid);
+      const agentIndex = up.findIndex((x) => AGENT.test(base(x.command)));
+      const lineage = agentIndex >= 0 ? up.slice(0, agentIndex + 1) : up.slice(0, 3);
+      for (const x of lineage) need.add(x.pid);
+      plan.set(pid, { cacheKey, lineage, agent: agentIndex >= 0 ? up[agentIndex] : null });
+    }
+    const env = need.size ? await this.#environments([...need]) : new Map();
+    for (const [pid, p] of plan) {
+      if (!this.sessionCache.has(p.cacheKey)) {
+        let found = null;
+        for (const x of p.lineage) { found = env.get(x.pid); if (found) break; }
+        const harness = p.agent ? (HARNESS[base(p.agent.command)] ?? base(p.agent.command)) : found?.harness ?? null;
+        const session = found ? { harness: found.harness ?? harness, id: found.id, address: found.address ?? `sc:${this.machine}:${found.harness ?? harness}:${found.id}`, agentPid: p.agent?.pid ?? null }
+          : p.agent ? { harness, id: null, address: null, agentPid: p.agent.pid } : null;
+        this.sessionCache.set(p.cacheKey, session);
+        if (this.sessionCache.size > 5000) this.sessionCache.delete(this.sessionCache.keys().next().value);
+      }
+      const session = this.sessionCache.get(p.cacheKey);
+      for (const v of views) if (v.pid === pid) v.session = session;
+    }
+  }
+
+  /** The session ids a process inherited, and nothing else from its environment. */
+  async #environments(pids) {
+    const map = new Map();
+    const pick = (text) => {
+      const supercode = /(?:^|\s|\0)SUPERCODE_CALLER=(sc:[^\s\0]+)/.exec(text)?.[1];
+      const claude = /(?:^|\s|\0)CLAUDE_CODE_SESSION_ID=([0-9a-f-]{36})/.exec(text)?.[1];
+      const codex = /(?:^|\s|\0)CODEX_THREAD_ID=([0-9a-zA-Z-]{8,})/.exec(text)?.[1];
+      if (claude) return { harness: 'claude-code', id: claude };
+      if (codex) return { harness: 'codex', id: codex };
+      if (supercode) { const [, , harness, id] = supercode.split(':'); return { harness, id, address: supercode }; }
+      return null;
+    };
+    if (this.platform === 'linux') {
+      for (const pid of pids) { try { const found = pick(readFileSync(`/proc/${pid}/environ`, 'utf8')); if (found) map.set(pid, found); } catch { /* another user's */ } }
+      return map;
+    }
+    const r = await run('/bin/ps', ['eww', '-o', 'pid=,command=', '-p', pids.join(',')], { timeoutMs: 10_000 });
+    for (const line of r.stdout.split('\n')) {
+      const m = /^\s*(\d+)\s(.*)$/.exec(line);
+      if (!m) continue;
+      const found = pick(m[2]);
+      if (found) map.set(Number(m[1]), found);
+    }
+    return map;
+  }
+}

package/lib/thresholds.mjs

@@ -0,0 +1,103 @@
+// Default alert windows (docs/adr/0006-machine-health-pack.md has each one's basis). A machine's config
+// overrides any value: `alarms.<name>.<field>` merges over these, and `volumes.<mount>` sets per-volume lines.
+export const DEFAULTS = {
+  intervalSec: 15,          // fast readings (Prometheus's common scrape interval)
+  slowIntervalSec: 60,      // disk space, services, the connector, tabs
+  footprintTopEverySec: 300, // macOS: every user's largest footprints through top (about 2 s of top per run)
+  repeatMin: 60,            // a raised alarm mails again this often (netdata's repeat cap)
+  historyHours: 24,
+  daemons: {
+    darwin: ['dasd', 'fseventsd', 'mds', 'mds_stores', 'mdworker_shared', 'WindowServer', 'kernel_task', 'backupd', 'softwareupdated', 'photoanalysisd', 'cloudd', 'bird', 'nsurlsessiond', 'trustd', 'syspolicyd', 'XprotectService', 'logd', 'runningboardd', 'launchservicesd', 'coreduetd', 'spotlightknowledged'],
+    linux: ['systemd-journald', 'kswapd0', 'dockerd', 'containerd', 'snapd', 'packagekitd', 'tracker-miner-fs-3', 'baloo_file', 'updatedb'],
+  },
+  alarms: {
+    'memory.pressure': { forSec: 30, critForSec: 0 },
+    'memory.free': { below: true, warn: 20, crit: 10, clear: 25, forSec: 30 },
+    'memory.swapping': { warn: 0, forSec: 15 },
+    'memory.swapped30m': { crit: 5 },
+    'memory.compressions': { warn: 30_000, clear: 15_000, forSec: 30 },
+    'load.perCore': { warn: 2, crit: 8, forSec: 900, critForSec: 60 },
+    'process.cpu': { warn: 100, crit: 200, clear: 50, forSec: 600 },
+    'renderer.cpu': { warn: 80, clear: 40, forSec: 600 },
+    'renderer.growth': { minMB: 2048, mbPerMin: 500 },
+    'process.growth': { minMB: 4096, mbPerMin: 1024 },
+    'browser.orphan': { forSec: 600 },
+    'daemon.cpu': { warn: 40, crit: 100, clear: 20, forSec: 120 },
+    'daemon.footprint': { warnMB: 4096, perDaemonMB: { fseventsd: 1024 } },
+    'disk.free': { below: true, warnPct: 10, critPct: 5, forSec: 1800 },
+    'disk.filling': { warnHours: 24, warnBelowPct: 40, critHours: 4, lookbackMin: 60 },
+    'disk.writer': { warn: 50, forSec: 300 },
+    'tcp.ephemeral': { warn: 50, crit: 70, forSec: 30 },
+    'tcp.timeWaitTarget': { warn: 25, crit: 50, forSec: 30 },
+    'files.open': { warn: 70, crit: 90, forSec: 900 },
+    'service.down': { forSec: 60 },
+    'supercode.connector': { forSec: 60 },
+    'supercode.launch': { stuckMin: 5, failedWithinMin: 30 },
+    'browser.tabs': { warn: null },
+    'power.thermal': { forSec: 60 },
+  },
+};
+
+const isObject = (v) => v && typeof v === 'object' && !Array.isArray(v);
+export function merge(base, over) {
+  if (!isObject(over)) return over === undefined ? base : over;
+  const out = { ...base };
+  for (const [k, v] of Object.entries(over)) out[k] = isObject(v) && isObject(base?.[k]) ? merge(base[k], v) : v;
+  return out;
+}
+
+/** Which levels a numeric value meets under a threshold definition. */
+export function meets(value, def) {
+  if (value == null || Number.isNaN(value)) return { warn: false, crit: false, clears: true };
+  const past = (line) => line != null && (def.below ? value < line : value > line);
+  const crit = past(def.crit);
+  const warn = crit || past(def.warn);
+  const clears = def.clear == null ? !warn : (def.below ? value >= def.clear : value <= def.clear);
+  return { warn, crit, clears };
+}
+
+const RANK = { none: 0, warn: 1, crit: 2 };
+
+/**
+ * One step of every alarm. `observations`: [{ key, family, def, value, met?, unit, summary, detail }] for the
+ * families read this cycle. A key of an evaluated family that was not observed has cleared (its entity is gone).
+ * Returns the transitions; `state` (key → record) is updated in place.
+ */
+export function judge(state, observations, families, now) {
+  const transitions = [];
+  const seen = new Set();
+  for (const o of observations) {
+    seen.add(o.key);
+    const m = o.met ?? meets(o.value, o.def);
+    const s = state[o.key] ?? (state[o.key] = { key: o.key, family: o.family, level: 'none', warnSince: null, critSince: null, raisedAt: null, lastMailAt: null });
+    s.warnSince = m.warn ? (s.warnSince ?? now) : null;
+    s.critSince = m.crit ? (s.critSince ?? now) : null;
+    const warnFor = (o.def.forSec ?? 0) * 1000;
+    const critFor = (o.def.critForSec ?? o.def.forSec ?? 0) * 1000;
+    let target = 'none';
+    if (m.crit && now - s.critSince >= critFor) target = 'crit';
+    else if (m.warn && now - s.warnSince >= warnFor) target = 'warn';
+    // Hysteresis: a raised alarm holds at warn until the value is past its clear line.
+    if (target === 'none' && s.level !== 'none' && !m.clears) target = 'warn';
+    Object.assign(s, { value: o.value, unit: o.unit ?? null, summary: o.summary, detail: o.detail ?? null, at: now });
+    if (target !== s.level) {
+      transitions.push({ key: o.key, from: s.level, to: target, value: o.value, unit: o.unit ?? null, summary: o.summary, detail: o.detail ?? null });
+      if (RANK[target] > RANK[s.level] && s.level === 'none') s.raisedAt = now;
+      s.level = target;
+      if (target === 'none') s.raisedAt = null;
+    }
+  }
+  for (const [key, s] of Object.entries(state)) {
+    if (seen.has(key) || !families.has(s.family)) continue;
+    // An alarm restored from before a restart is not cleared by the first samples, which have no rates yet.
+    if (s.restoredUntil && now < s.restoredUntil) continue;
+    if (s.level !== 'none') transitions.push({ key, from: s.level, to: 'none', value: null, summary: `${s.summary} — cleared: no longer over its line, or the process exited`, detail: s.detail, gone: true });
+    delete state[key];
+  }
+  return transitions;
+}
+
+export function raised(state) {
+  return Object.values(state).filter((s) => s.level !== 'none').sort((a, b) => RANK[b.level] - RANK[a.level] || a.key.localeCompare(b.key))
+    .map(({ key, family, level, value, unit, summary, detail, raisedAt }) => ({ key, family, level, value, unit, summary, detail, since: raisedAt ? new Date(raisedAt).toISOString() : null }));
+}

package/lib/win32.mjs

@@ -0,0 +1,37 @@
+// Windows readers: what Node and netstat give without a native module. Other features report themselves off.
+import { freemem, totalmem } from 'node:os';
+import { statfsSync } from 'node:fs';
+import { run } from './run.mjs';
+
+export async function memory() {
+  const total = totalmem();
+  return { pressureLevel: null, totalBytes: total, freePct: Math.round((freemem() / total) * 100), swapUsedBytes: null, swapTotalBytes: null, counters: {}, pageSize: 4096 };
+}
+
+function splitAddress(text) {
+  const at = text.lastIndexOf(':');
+  return { addr: text.slice(0, at).replace(/^\[|\]$/g, ''), port: Number(text.slice(at + 1)) || 0 };
+}
+export async function tcp() {
+  const r = await run('netstat', ['-an', '-p', 'TCP'], { timeoutMs: 15_000 });
+  if (!r.ok) throw new Error(r.error);
+  const sockets = [];
+  for (const line of r.stdout.split(/\r?\n/)) {
+    const f = line.trim().split(/\s+/);
+    if (f[0] !== 'TCP' || f.length < 4) continue;
+    sockets.push({ local: splitAddress(f[1]), foreign: splitAddress(f[2]), state: f[3] === 'LISTENING' ? 'LISTEN' : f[3] });
+  }
+  // The Windows default dynamic range (netsh int ipv4 show dynamicport tcp): 49152-65535, 120 s TIME_WAIT.
+  return { sockets, range: { first: 49152, last: 65535 }, timeWaitSec: 120 };
+}
+
+export async function volumes() {
+  const rows = [];
+  for (const letter of 'CDEFGHIJ') {
+    try { const s = statfsSync(`${letter}:\\`); rows.push({ mount: `${letter}:`, device: `${letter}:`, totalBytes: s.blocks * s.bsize, freeBytes: s.bavail * s.bsize }); } catch { /* no such drive */ }
+  }
+  return rows;
+}
+
+export const isRoot = () => false;
+export const powermetricsAvailable = () => false;