@volter/supercode-health 0.0.0-stage → 0.1.1

package/lib/mail.mjs

@@ -0,0 +1,64 @@
+// Alarm mail through supercode's public mailbox door (`supercode message remote-deliver`), from
+// `sc:<machine>:operator:machine-health`. Undelivered mail stays in the outbox and is retried each cycle,
+// as the machine-events outbox does, until it is a day old.
+import { spawn } from 'node:child_process';
+import { readFileSync, writeFileSync, renameSync } from 'node:fs';
+
+export class Outbox {
+  constructor({ file, machine, env = process.env }) {
+    Object.assign(this, { file, machine, env });
+    try { this.items = JSON.parse(readFileSync(file, 'utf8')); } catch { this.items = []; }
+    this.error = null;
+  }
+  save() { const temp = `${this.file}.${process.pid}.tmp`; writeFileSync(temp, JSON.stringify(this.items) + '\n', { mode: 0o600 }); renameSync(temp, this.file); }
+  enqueue(to, id, subject, body) { this.items.push({ to, id, subject, body, at: Date.now(), tries: 0 }); this.save(); }
+  async flush() {
+    for (const item of [...this.items]) {
+      if (Date.now() - item.at > 24 * 3_600_000) { this.items = this.items.filter((x) => x !== item); this.error = `dropped undelivered ${item.id} to ${item.to} after a day`; continue; }
+      try { await this.deliver(item); this.items = this.items.filter((x) => x !== item); this.error = null; }
+      catch (error) { item.tries += 1; this.error = `mail to ${item.to}: ${error.message}`; }
+    }
+    this.save();
+    return { pending: this.items.length, error: this.error };
+  }
+  deliver(item) {
+    return new Promise((resolve, reject) => {
+      const child = spawn(this.env.SUPERCODE_BIN || 'supercode', ['message', 'remote-deliver'], { env: this.env, stdio: ['pipe', 'pipe', 'pipe'], windowsHide: true, shell: process.platform === 'win32' });
+      let detail = '';
+      const timer = setTimeout(() => { child.kill('SIGTERM'); reject(new Error('delivery timed out; kept for retry')); }, 30_000);
+      child.stdout.on('data', (c) => { detail = (detail + c).slice(-2000); });
+      child.stderr.on('data', (c) => { detail = (detail + c).slice(-2000); });
+      child.once('error', (error) => { clearTimeout(timer); reject(error); });
+      child.once('close', (code) => {
+        clearTimeout(timer);
+        let receipt; try { receipt = JSON.parse(detail.trim().split('\n').at(-1)); } catch { /* none */ }
+        if (code === 0 && receipt?.code === 0) resolve(receipt); else reject(new Error(receipt?.text || detail.trim().slice(-300) || `exit ${code}`));
+      });
+      child.stdin.on('error', () => {});
+      const board = item.to.split(':')[2] === 'board';
+      child.stdin.end(JSON.stringify({ op: 'send', from: `sc:${this.machine}:operator:machine-health`, from_name: 'Machine health', to: item.to, id: item.id, subject: item.subject, queue: board, body: item.body }));
+    });
+  }
+}
+
+const LEVEL = { warn: 'WARN', crit: 'CRIT', none: 'clear' };
+function line(t) {
+  const d = t.detail;
+  const who = d?.session?.address ? `; session ${d.session.address}` : d?.session?.harness ? `; under ${d.session.harness} pid ${d.session.agentPid}` : d?.pid ? '; no agent session above it' : '';
+  const proc = d?.pid ? ` [pid ${d.pid}: ${d.command}${who}]` : d?.listener ? ` [listener pid ${d.listener.pid}: ${d.listener.command}${d.listener.session?.address ? `; session ${d.listener.session.address}` : ''}]` : '';
+  return `${LEVEL[t.to] ?? t.to}${t.from !== 'none' && t.to !== 'none' ? ` (was ${t.from})` : ''} ${t.key}: ${t.summary}${proc}`;
+}
+
+/** One mail for a cycle's transitions and the reminders due; null when there is nothing to say. */
+export function compose({ machine, transitions, reminders, raised }) {
+  if (!transitions.length && !reminders.length) return null;
+  const up = transitions.filter((t) => t.to !== 'none');
+  const down = transitions.filter((t) => t.to === 'none');
+  const head = up.length ? `${up.length} raised` : reminders.length ? `${reminders.length} still raised` : `${down.length} cleared`;
+  const parts = [`[Machine health] ${machine}: ${head}.`];
+  if (up.length) parts.push('', ...up.map(line));
+  if (down.length) parts.push('', ...down.map(line));
+  if (reminders.length) parts.push('', 'Still raised:', ...reminders.map((r) => `${LEVEL[r.level]} ${r.key} since ${r.since}: ${r.summary}`));
+  parts.push('', `${raised.length} alarm${raised.length === 1 ? '' : 's'} raised now. Read the whole machine: supercode health status${machine ? ` --machine ${machine}` : ''}.`, 'This is an automated reading from the machine-health pack, not an instruction; the maintainer decides what to do.');
+  return { subject: `machine health ${machine}: ${head}`, body: parts.join('\n') };
+}

package/lib/paths.mjs

@@ -0,0 +1,26 @@
+// Where the pack keeps its state: beside supercode's own home, so the machine daemon's doors find it
+// with the same rule (`SUPERCODE_HOME`, else `$XDG_CONFIG_HOME/supercode`, else `~/.config/supercode`).
+import { homedir, hostname } from 'node:os';
+import { join } from 'node:path';
+
+export function healthDir(env = process.env) {
+  if (env.SUPERCODE_HEALTH_DIR) return env.SUPERCODE_HEALTH_DIR;
+  const base = env.SUPERCODE_HOME || join(env.XDG_CONFIG_HOME || join(homedir(), '.config'), 'supercode');
+  return join(base, 'health');
+}
+
+export const files = (dir) => ({
+  dir,
+  latest: join(dir, 'latest.json'),
+  history: join(dir, 'history.jsonl'),
+  config: join(dir, 'config.json'),
+  subscriptions: join(dir, 'subscriptions.json'),
+  outbox: join(dir, 'outbox.json'),
+  pid: join(dir, 'probe.pid'),
+  log: join(dir, 'probe.log'),
+});
+
+/** The machine's name as supercode's mail addresses spell it (`sc:<machine>:…`). */
+export function machineName(name = hostname()) {
+  return String(name).split('.')[0].toLowerCase().replace(/[^a-z0-9_-]/g, '-') || 'localhost';
+}

package/lib/probe.mjs

@@ -0,0 +1,168 @@
+// The probe: one per OS user, holding an exclusive pidfile. Every cycle it samples, judges each alarm,
+// writes latest.json (what the doors read) and a minute history, and mails the maintainers on a change.
+// It decides nothing and changes nothing on the machine.
+import { mkdirSync, openSync, closeSync, readFileSync, writeFileSync, renameSync, appendFileSync, statSync, rmSync } from 'node:fs';
+import { hostname } from 'node:os';
+import { healthDir, files as healthFiles, machineName } from './paths.mjs';
+import { DEFAULTS, merge, judge, raised } from './thresholds.mjs';
+import { resolveFeatures } from './features.mjs';
+import { Sampler } from './sample.mjs';
+import { Outbox, compose } from './mail.mjs';
+import { run } from './run.mjs';
+
+export async function platformReader(platform = process.platform) {
+  if (platform === 'darwin') return import('./darwin.mjs');
+  if (platform === 'linux') return import('./linux.mjs');
+  if (platform === 'win32') return import('./win32.mjs');
+  return {};
+}
+
+const readJson = (file, fallback) => { try { return JSON.parse(readFileSync(file, 'utf8')); } catch { return fallback; } };
+function writeAtomic(file, value) { const temp = `${file}.${process.pid}.tmp`; writeFileSync(temp, JSON.stringify(value, null, 2) + '\n', { mode: 0o600 }); renameSync(temp, file); }
+
+export function loadConfig(file) {
+  const raw = readJson(file, {});
+  return { ...merge(DEFAULTS, raw), source: raw && Object.keys(raw).length ? file : null };
+}
+
+/** The addresses alarms go to: the config's maintainers and every door subscription. */
+export function subscribers(f, config) {
+  const subs = readJson(f.subscriptions, { addresses: [] }).addresses ?? [];
+  return [...new Set([...(config.maintainers ?? []), ...subs])];
+}
+
+export function subscribe(dir, address, remove = false) {
+  if (!/^sc:[^:\s]+:[^:\s]+:\S+$/.test(address ?? '')) throw new Error('subscribe a full sc:<machine>:<harness>:<id> address');
+  const f = healthFiles(dir);
+  mkdirSync(dir, { recursive: true, mode: 0o700 });
+  const current = readJson(f.subscriptions, { addresses: [] }).addresses ?? [];
+  const addresses = current.filter((a) => a !== address);
+  if (!remove) addresses.push(address);
+  writeAtomic(f.subscriptions, { addresses });
+  return { addresses };
+}
+
+export class Probe {
+  constructor({ env = process.env, machine, dir = healthDir(env), platform = process.platform } = {}) {
+    this.env = env;
+    this.platform = platform;
+    this.dir = dir;
+    this.f = healthFiles(dir);
+    this.machine = machineName(machine || env.SUPERCODE_HEALTH_MACHINE || hostname());
+    this.state = {};
+    this.lastHistory = 0;
+    this.sampleMs = null;
+  }
+
+  /** Claim the one probe per user. Returns false when a live probe holds the pidfile. */
+  lock() {
+    mkdirSync(this.dir, { recursive: true, mode: 0o700 });
+    for (let attempt = 0; attempt < 2; attempt += 1) {
+      try { const fd = openSync(this.f.pid, 'wx', 0o600); writeFileSync(fd, `${process.pid}\n`); closeSync(fd); this.locked = true; return true; }
+      catch (error) {
+        if (error.code !== 'EEXIST') throw error;
+        const pid = Number(readFileSync(this.f.pid, 'utf8').trim());
+        let alive = false; try { process.kill(pid, 0); alive = pid !== process.pid; } catch (e) { alive = e.code === 'EPERM'; }
+        if (alive) { this.holder = pid; return false; }
+        rmSync(this.f.pid, { force: true });
+      }
+    }
+    return false;
+  }
+  unlock() { if (this.locked) { try { if (Number(readFileSync(this.f.pid, 'utf8')) === process.pid) rmSync(this.f.pid, { force: true }); } catch { /* gone */ } } }
+
+  async init() {
+    this.reader = await platformReader(this.platform);
+    this.config = loadConfig(this.f.config);
+    this.configMtime = this.#mtime(this.f.config);
+    this.sampler = new Sampler({ reader: this.reader, platform: this.platform, machine: this.machine, env: this.env, config: this.config });
+    this.outbox = new Outbox({ file: this.f.outbox, machine: this.machine, env: this.env });
+    this.extras = {
+      rusage: this.reader.rusageAvailable ? await this.reader.rusageAvailable() : false,
+      supercode: (await run(this.env.SUPERCODE_BIN || 'supercode', ['--version'], { timeoutMs: 10_000, env: this.env })).ok,
+    };
+    const previous = readJson(this.f.latest, null);
+    // Alarms raised before a restart stay raised (and are not re-mailed); their windows restart honestly.
+    const alive = (pid) => { try { process.kill(pid, 0); return true; } catch (error) { return error.code === 'EPERM'; } };
+    for (const a of (previous?.alarms ?? []).filter((x) => !x.detail?.pid || alive(x.detail.pid))) this.state[a.key] = { key: a.key, family: a.family, level: a.level, warnSince: null, critSince: null, raisedAt: a.since ? Date.parse(a.since) : Date.now(), lastMailAt: Date.now(), restoredUntil: Date.now() + 2 * 60_000, value: a.value, unit: a.unit, summary: a.summary, detail: a.detail };
+  }
+
+  #mtime(file) { try { return statSync(file).mtimeMs; } catch { return null; } }
+
+  features() {
+    const root = this.reader.isRoot?.() ?? false;
+    return resolveFeatures({ platform: this.platform, reader: this.reader, config: this.config, root, extras: this.extras });
+  }
+
+  async cycle(now = Date.now()) {
+    const mtime = this.#mtime(this.f.config);
+    if (mtime !== this.configMtime) { this.config = loadConfig(this.f.config); this.sampler.config = this.config; this.configMtime = mtime; }
+    const features = this.features();
+    const t0 = Date.now();
+    const { readings, observations, families, errors } = await this.sampler.sample(features, now);
+    this.sampleMs = Date.now() - t0;
+    const transitions = judge(this.state, observations, families, now);
+    const alarms = raised(this.state);
+    const repeatMs = this.config.repeatMin * 60_000;
+    const reminders = transitions.length ? [] : alarms.filter((a) => now - (this.state[a.key].lastMailAt ?? 0) >= repeatMs);
+    const to = subscribers(this.f, this.config);
+    const mail = compose({ machine: this.machine, transitions, reminders, raised: alarms });
+    if (mail) {
+      for (const a of alarms) this.state[a.key].lastMailAt = now;
+      for (const address of to) this.outbox.enqueue(address, `health-${this.machine}-${now}-${address.split(':').at(-1).slice(0, 12)}`, mail.subject, mail.body);
+      this.log(`${mail.subject}${to.length ? '' : ' (no maintainer subscribed)'}\n${mail.body}`);
+    }
+    const delivery = this.outbox.items.length ? await this.outbox.flush() : { pending: 0, error: this.outbox.error };
+    const snapshot = {
+      version: 1, machine: this.machine, platform: this.platform, at: new Date(now).toISOString(),
+      probe: { pid: process.pid, intervalSec: this.config.intervalSec, slowIntervalSec: this.config.slowIntervalSec, sampleMs: this.sampleMs, user: this.env.USER ?? null, root: this.reader.isRoot?.() ?? false, config: this.config.source },
+      features: features.map(({ id, on, barrier, reason, reads }) => ({ id, on, barrier, reason, reads, ...(errors[id] ? { error: errors[id] } : {}) })),
+      alarms,
+      transitions: transitions.map((t) => ({ key: t.key, from: t.from, to: t.to, summary: t.summary })),
+      maintainers: to,
+      mail: delivery,
+      readings,
+    };
+    writeAtomic(this.f.latest, snapshot);
+    if (now - this.lastHistory >= 60_000) { this.#history(snapshot, now); this.lastHistory = now; }
+    return snapshot;
+  }
+
+  #history(s, now) {
+    const r = s.readings;
+    const row = {
+      at: s.at, load1: r.load?.one, freePct: r.memory?.freePct, pressure: r.memory?.pressureLevel, swapoutsPerSec: r.memory?.swapoutsPerSec, compressionsPerSec: r.memory?.compressionsPerSec,
+      timeWait: r.tcp?.timeWait, ephemeralPct: r.tcp?.ephemeral?.[0]?.pct, renderers: r.renderers?.count,
+      disk: r.disk?.map((d) => [d.mount, d.freeGB]), topCpu: r.processes?.topCpu?.slice(0, 3).map((p) => [p.pid, p.cpuPct, p.command.slice(0, 60)]),
+      // Every process over 1 GB with its growth, so a process that is gone can still be judged leak or live work.
+      big: r.processes?.topMemory?.filter((p) => p.footprintMB >= 1024).map((p) => [p.pid, p.footprintMB, p.growthMBPerMin, p.command.slice(0, 100), p.session?.address ?? null]),
+      alarms: s.alarms.map((a) => `${a.level}:${a.key}`),
+    };
+    appendFileSync(this.f.history, JSON.stringify(row) + '\n', { mode: 0o600 });
+    // Trim to the configured window once an hour.
+    if (!this.trimAt || now - this.trimAt > 3_600_000) {
+      this.trimAt = now;
+      const keep = this.config.historyHours * 60;
+      const lines = readFileSync(this.f.history, 'utf8').split('\n').filter(Boolean);
+      if (lines.length > keep) writeFileSync(this.f.history, lines.slice(-keep).join('\n') + '\n', { mode: 0o600 });
+    }
+  }
+
+  log(text) {
+    try {
+      appendFileSync(this.f.log, `${new Date().toISOString()} ${text}\n`, { mode: 0o600 });
+      if (statSync(this.f.log).size > 5 * 1024 * 1024) { const lines = readFileSync(this.f.log, 'utf8').split('\n'); writeFileSync(this.f.log, lines.slice(-5000).join('\n')); }
+    } catch { /* logging never stops the probe */ }
+  }
+
+  async run({ signal } = {}) {
+    await this.init();
+    this.log(`probe ${process.pid} started on ${this.machine} (${this.platform}); features on: ${this.features().filter((f) => f.on).map((f) => f.id).join(', ')}`);
+    while (!signal?.aborted) {
+      const started = Date.now();
+      try { await this.cycle(started); } catch (error) { this.log(`cycle failed: ${error.stack ?? error.message}`); }
+      const wait = Math.max(1000, this.config.intervalSec * 1000 - (Date.now() - started));
+      await new Promise((resolve) => { const t = setTimeout(resolve, wait); signal?.addEventListener('abort', () => { clearTimeout(t); resolve(); }, { once: true }); });
+    }
+  }
+}

package/lib/run.mjs

@@ -0,0 +1,33 @@
+// One bounded child process. Every instrument goes through here: a reading that cannot finish in its budget
+// is an error the feature reports, never a hang of the probe.
+import { execFile } from 'node:child_process';
+
+export function run(file, args = [], { timeoutMs = 10_000, env, input, maxBuffer = 64 * 1024 * 1024 } = {}) {
+  return new Promise((resolve) => {
+    const child = execFile(file, args, { timeout: timeoutMs, env, maxBuffer, windowsHide: true, encoding: 'utf8', shell: process.platform === 'win32' }, (error, stdout, stderr) => {
+      if (error) resolve({ ok: false, stdout: stdout ?? '', stderr: stderr ?? '', error: error.killed ? `${file} timed out after ${timeoutMs} ms` : (error.code === 'ENOENT' ? `${file} not found` : (stderr || error.message).trim().slice(0, 300)) });
+      else resolve({ ok: true, stdout, stderr });
+    });
+    if (input !== undefined) child.stdin.end(input);
+  });
+}
+
+/** `[dd-]hh:mm:ss[.cc]` or `mm:ss[.cc]` (ps's TIME and ETIME) in seconds. */
+export function clockSeconds(text) {
+  if (!text) return 0;
+  let days = 0, rest = String(text).trim();
+  const dash = rest.indexOf('-');
+  if (dash > 0) { days = Number(rest.slice(0, dash)); rest = rest.slice(dash + 1); }
+  const parts = rest.split(':').map(Number);
+  let seconds = 0;
+  for (const part of parts) seconds = seconds * 60 + (Number.isFinite(part) ? part : 0);
+  return days * 86400 + seconds;
+}
+
+/** top's MEM column (`1234M`, `2.1G`, `512K`, `96B`, with a trailing `+`/`-`) in bytes. */
+export function topBytes(text) {
+  const match = /^([\d.]+)([BKMGT]?)[+-]?$/i.exec(String(text ?? '').trim());
+  if (!match) return null;
+  const unit = { B: 1, K: 1024, M: 1024 ** 2, G: 1024 ** 3, T: 1024 ** 4 }[(match[2] || 'B').toUpperCase()];
+  return Math.round(Number(match[1]) * unit);
+}

package/lib/rusage.py

@@ -0,0 +1,17 @@
+# One pass of libproc's proc_pid_rusage over every pid this user may read (other users' fail without root).
+# Prints "pid phys_footprint diskio_read diskio_written" per readable pid. About 50 ms for 1,100 pids.
+import ctypes
+lib = ctypes.CDLL('/usr/lib/libproc.dylib')
+class RU(ctypes.Structure):
+    _fields_ = [('uuid', ctypes.c_uint8 * 16)] + [(n, ctypes.c_uint64) for n in (
+        'user system pkg_idle int_wkups pageins wired resident phys proc_start proc_exit child_user child_system '
+        'child_pkg_idle child_wkups child_pageins child_elapsed diskio_read diskio_written').split()]
+n = lib.proc_listallpids(None, 0)
+pids = (ctypes.c_int * (n + 64))()
+n = lib.proc_listallpids(pids, ctypes.sizeof(pids))
+out = []
+for pid in pids[:n]:
+    r = RU()
+    if pid > 0 and lib.proc_pid_rusage(pid, 2, ctypes.byref(r)) == 0:
+        out.append('%d %d %d %d' % (pid, r.phys, r.diskio_read, r.diskio_written))
+print('\n'.join(out))