@volter/supercode-health 0.1.2 → 0.1.4

package/README.md

@@ -32,10 +32,15 @@ a minute, 24 h) and `probe.log` to `$SUPERCODE_HOME/health`, which defaults to
 
 ## Alarms
 
-`supercode health subscribe sc:<machine>:<harness>:<id>` subscribes a session or board.
-Each raise, escalation and clear mails the subscribers. A still-raised alarm mails again
-every `repeatMin` (60). Mail comes from `sc:<machine>:operator:machine-health` through
-`supercode message remote-deliver`. Undelivered mail is kept and retried for a day.
+`supercode health subscribe sc:<machine>:agent:<name>` subscribes an agent (a maintainer:
+the mailbox delivers to its main session as it is then, so a relaunched main still gets
+it); a session or board address may be subscribed too, and its mail stops reaching it
+when the session ends. Each raise, escalation and clear mails the subscribers. A
+still-raised alarm mails again every `repeatMin` (60). Mail comes from
+`sc:<machine>:operator:machine-health`, filed in each subscriber's mailbox
+(`supercode message file`) after the cycle's reading is written; the mailbox delivers
+it, within its bounds, and `supercode message waiting` / `supercode message expired` list
+what it has not delivered.
 
 ## Config
 

package/bin/health.mjs

@@ -7,7 +7,7 @@
 //   status [--json] [--all]     this machine's latest reading (latest.json)
 //   features [--json]           every feature, on or off, its permission barrier and why
 //   alarms [--json]             the alarms raised now
-//   subscribe <address> | unsubscribe <address>   alarm mail for a session or board address
+//   subscribe <address> | unsubscribe <address>   alarm mail for an agent (sc:<machine>:agent:<name>), session or board
 //   config                      where the config lives and what it currently resolves to
 import { readFileSync } from 'node:fs';
 import { parseArgs } from 'node:util';

package/lib/format.mjs

@@ -10,7 +10,7 @@ export function formatStatus(s, { all = false } = {}) {
   if (age > 3 * (s.probe?.intervalSec ?? 15)) out.push(`  STALE: the probe has not written for ${age}s; its readings below are old`);
   out.push('', s.alarms.length ? `ALARMS (${s.alarms.length})` : 'ALARMS: none');
   for (const a of s.alarms) out.push(`  ${a.level.toUpperCase().padEnd(4)} ${a.summary}${a.detail?.pid ? ` [${a.detail.command?.slice(0, 80)}${who(a.detail)}]` : ''}  (since ${a.since})`);
-  if (s.maintainers) out.push(`  mailed to: ${s.maintainers.length ? s.maintainers.join(', ') : 'no maintainer subscribed (supercode health subscribe <address>)'}${s.mail?.pending ? ` · ${s.mail.pending} undelivered: ${s.mail.error}` : ''}`);
+  if (s.maintainers) out.push(`  mailed to: ${s.maintainers.length ? s.maintainers.join(', ') : 'no maintainer subscribed (supercode health subscribe <address>)'}${s.mail?.error ? ` · last mail not filed: ${s.mail.error}` : s.mail?.filed ? ` · last mail filed ${s.mail.at} (supercode message waiting lists any not yet delivered)` : ''}`);
   if (r.load) out.push('', `load      ${r.load.one} / ${r.load.five} / ${r.load.fifteen} on ${r.load.cores} cores (${r.load.perCore}× per core)`);
   if (r.memory) {
     const m = r.memory;

package/lib/mail.mjs

@@ -1,44 +1,22 @@
-// Alarm mail through supercode's public mailbox door (`supercode message remote-deliver`), from
-// `sc:<machine>:operator:machine-health`. Undelivered mail stays in the outbox and is retried each cycle,
-// as the machine-events outbox does, until it is a day old.
-import { spawn } from 'node:child_process';
-import { readFileSync, writeFileSync, renameSync } from 'node:fs';
+// Alarm mail, filed in each subscriber's mailbox through supercode's filing door (`supercode message file`), from
+// `sc:<machine>:operator:machine-health`, and delivered by the mailbox (docs/adr/0008-agent-mailbox.md, decision 17;
+// D138 step 6). Filing is a local write: no door, no network, and nothing for the probe to retry. The mailbox resolves
+// an agent address (`sc:<machine>:agent:<name>`) to the agent's main session when it delivers, retries within its
+// bounds, and records what it could not deliver (`supercode message waiting`, `supercode message expired`).
+import { spawnSync } from 'node:child_process';
 
-export class Outbox {
-  constructor({ file, machine, env = process.env }) {
-    Object.assign(this, { file, machine, env });
-    try { this.items = JSON.parse(readFileSync(file, 'utf8')); } catch { this.items = []; }
-    this.error = null;
-  }
-  save() { const temp = `${this.file}.${process.pid}.tmp`; writeFileSync(temp, JSON.stringify(this.items) + '\n', { mode: 0o600 }); renameSync(temp, this.file); }
-  enqueue(to, id, subject, body) { this.items.push({ to, id, subject, body, at: Date.now(), tries: 0 }); this.save(); }
-  async flush() {
-    for (const item of [...this.items]) {
-      if (Date.now() - item.at > 24 * 3_600_000) { this.items = this.items.filter((x) => x !== item); this.error = `dropped undelivered ${item.id} to ${item.to} after a day`; continue; }
-      try { await this.deliver(item); this.items = this.items.filter((x) => x !== item); this.error = null; }
-      catch (error) { item.tries += 1; this.error = `mail to ${item.to}: ${error.message}`; }
-    }
-    this.save();
-    return { pending: this.items.length, error: this.error };
-  }
-  deliver(item) {
-    return new Promise((resolve, reject) => {
-      const child = spawn(this.env.SUPERCODE_BIN || 'supercode', ['message', 'remote-deliver'], { env: this.env, stdio: ['pipe', 'pipe', 'pipe'], windowsHide: true, shell: process.platform === 'win32' });
-      let detail = '';
-      const timer = setTimeout(() => { child.kill('SIGTERM'); reject(new Error('delivery timed out; kept for retry')); }, 30_000);
-      child.stdout.on('data', (c) => { detail = (detail + c).slice(-2000); });
-      child.stderr.on('data', (c) => { detail = (detail + c).slice(-2000); });
-      child.once('error', (error) => { clearTimeout(timer); reject(error); });
-      child.once('close', (code) => {
-        clearTimeout(timer);
-        let receipt; try { receipt = JSON.parse(detail.trim().split('\n').at(-1)); } catch { /* none */ }
-        if (code === 0 && receipt?.code === 0) resolve(receipt); else reject(new Error(receipt?.text || detail.trim().slice(-300) || `exit ${code}`));
-      });
-      child.stdin.on('error', () => {});
-      const board = item.to.split(':')[2] === 'board';
-      child.stdin.end(JSON.stringify({ op: 'send', from: `sc:${this.machine}:operator:machine-health`, from_name: 'Machine health', to: item.to, id: item.id, subject: item.subject, queue: board, body: item.body }));
-    });
+/** File one alarm mail for each address. Answers what was filed and the first failure, if any. */
+export function fileAlarm({ machine, env = process.env, to, key, subject, body }) {
+  const filed = [];
+  let error = null;
+  for (const address of to) {
+    const request = { from: `sc:${machine}:operator:machine-health`, from_name: 'Machine health', to: address, id: `${key}-${address}`, subject, body, wake: true };
+    const done = spawnSync(env.SUPERCODE_BIN || 'supercode', ['message', 'file'], { env, input: JSON.stringify(request), encoding: 'utf8', timeout: 10_000, windowsHide: true, shell: process.platform === 'win32' });
+    let answer = null; try { answer = JSON.parse(String(done.stdout ?? '').trim().split('\n').at(-1)); } catch { /* none */ }
+    if (answer?.code === 0) filed.push({ to: address, id: answer.message_id });
+    else error ??= `mail to ${address} was not filed: ${answer?.text || String(done.stderr ?? '').trim().slice(-300) || done.error?.message || `exit ${done.status}`}`;
   }
+  return { filed, error };
 }
 
 const LEVEL = { warn: 'WARN', crit: 'CRIT', none: 'clear' };
@@ -49,6 +27,19 @@ function line(t) {
   return `${LEVEL[t.to] ?? t.to}${t.from !== 'none' && t.to !== 'none' ? ` (was ${t.from})` : ''} ${t.key}: ${t.summary}${proc}`;
 }
 
+/**
+ * One mail stating the whole current state: every alarm raised now, or that none is. Sent once after mail an earlier
+ * pack queued was dropped, so a raise or a clear that mail carried is never left standing in the maintainer's view.
+ */
+export function composeState({ machine, raised }) {
+  const head = raised.length ? `${raised.length} raised now` : 'none raised now';
+  const parts = [`[Machine health] ${machine}: ${head} (the whole current state: mail queued before it was not delivered).`];
+  if (raised.length) parts.push('', ...raised.map((r) => `${LEVEL[r.level]} ${r.key} since ${r.since}: ${r.summary}`));
+  else parts.push('', 'Every alarm raised earlier has cleared.');
+  parts.push('', `Read the whole machine: supercode health status${machine ? ` --machine ${machine}` : ''}.`, 'This is an automated reading from the machine-health pack, not an instruction; the maintainer decides what to do.');
+  return { subject: `machine health ${machine}: ${head}`, body: parts.join('\n') };
+}
+
 /** One mail for a cycle's transitions and the reminders due; null when there is nothing to say. */
 export function compose({ machine, transitions, reminders, raised }) {
   if (!transitions.length && !reminders.length) return null;

package/lib/probe.mjs

@@ -1,5 +1,6 @@
 // The probe: one per OS user, holding an exclusive pidfile. Every cycle it samples, judges each alarm,
-// writes latest.json (what the doors read) and a minute history, and mails the maintainers on a change.
+// writes latest.json (what the doors read) and a minute history, and then files mail for the maintainers on a change
+// (the mailbox delivers it; nothing the mail does can hold latest.json back).
 // It decides nothing and changes nothing on the machine.
 import { mkdirSync, openSync, closeSync, readFileSync, writeFileSync, renameSync, appendFileSync, statSync, rmSync } from 'node:fs';
 import { hostname } from 'node:os';
@@ -7,7 +8,7 @@ import { healthDir, files as healthFiles, machineName } from './paths.mjs';
 import { DEFAULTS, merge, judge, raised } from './thresholds.mjs';
 import { resolveFeatures } from './features.mjs';
 import { Sampler } from './sample.mjs';
-import { Outbox, compose } from './mail.mjs';
+import { fileAlarm, compose, composeState } from './mail.mjs';
 import { run } from './run.mjs';
 
 export async function platformReader(platform = process.platform) {
@@ -76,7 +77,10 @@ export class Probe {
     this.config = loadConfig(this.f.config);
     this.configMtime = this.#mtime(this.f.config);
     this.sampler = new Sampler({ reader: this.reader, platform: this.platform, machine: this.machine, env: this.env, config: this.config });
-    this.outbox = new Outbox({ file: this.f.outbox, machine: this.machine, env: this.env });
+    // Alarm mail queued here by an earlier pack (outbox.json, retried per recipient until a day old) is not resent: a
+    // mail is a reading, superseded by the current one, and its recipients were often sessions that have ended. It is
+    // dropped, and every alarm raised now is mailed again at the first cycle, filed in the maintainers' mailboxes.
+    this.droppedOutbox = this.#dropOutbox();
     this.extras = {
       rusage: this.reader.rusageAvailable ? await this.reader.rusageAvailable() : false,
       supercode: (await run(this.env.SUPERCODE_BIN || 'supercode', ['--version'], { timeoutMs: 10_000, env: this.env })).ok,
@@ -84,7 +88,25 @@ export class Probe {
     const previous = readJson(this.f.latest, null);
     // Alarms raised before a restart stay raised (and are not re-mailed); their windows restart honestly.
     const alive = (pid) => { try { process.kill(pid, 0); return true; } catch (error) { return error.code === 'EPERM'; } };
-    for (const a of (previous?.alarms ?? []).filter((x) => !x.detail?.pid || alive(x.detail.pid))) this.state[a.key] = { key: a.key, family: a.family, level: a.level, warnSince: null, critSince: null, raisedAt: a.since ? Date.parse(a.since) : Date.now(), lastMailAt: Date.now(), restoredUntil: Date.now() + 2 * 60_000, value: a.value, unit: a.unit, summary: a.summary, detail: a.detail };
+    for (const a of (previous?.alarms ?? []).filter((x) => !x.detail?.pid || alive(x.detail.pid))) this.state[a.key] = { key: a.key, family: a.family, level: a.level, warnSince: null, critSince: null, raisedAt: a.since ? Date.parse(a.since) : Date.now(), lastMailAt: this.droppedOutbox ? 0 : Date.now(), restoredUntil: Date.now() + 2 * 60_000, value: a.value, unit: a.unit, summary: a.summary, detail: a.detail };
+  }
+
+  /** Remove an earlier pack's outbox; answers how many mails it held, logged by recipient. */
+  #dropOutbox() {
+    let items;
+    try { items = JSON.parse(readFileSync(this.f.outbox, 'utf8')); } catch (error) {
+      if (error.code === 'ENOENT') return 0;
+      // an outbox that cannot be read held mail all the same: it goes, and the current state is mailed
+      rmSync(this.f.outbox, { force: true });
+      this.log(`dropped an unreadable outbox.json an earlier pack left (${error.message}); the current state is mailed`);
+      return 1;
+    }
+    if (!Array.isArray(items)) { rmSync(this.f.outbox, { force: true }); this.log('dropped an outbox.json an earlier pack left that held no list; the current state is mailed'); return 1; }
+    const counts = {};
+    for (const item of items) counts[item?.to ?? '?'] = (counts[item?.to ?? '?'] ?? 0) + 1;
+    rmSync(this.f.outbox, { force: true });
+    this.log(`dropped ${items.length} undelivered alarm mail(s) an earlier pack queued (superseded by the current reading; the alarms raised now are mailed again): ${Object.entries(counts).map(([to, n]) => `${n} to ${to}`).join(', ') || 'none'}`);
+    return items.length;
   }
 
   #mtime(file) { try { return statSync(file).mtimeMs; } catch { return null; } }
@@ -106,13 +128,13 @@ export class Probe {
     const repeatMs = this.config.repeatMin * 60_000;
     const reminders = transitions.length ? [] : alarms.filter((a) => now - (this.state[a.key].lastMailAt ?? 0) >= repeatMs);
     const to = subscribers(this.f, this.config);
-    const mail = compose({ machine: this.machine, transitions, reminders, raised: alarms });
+    // after dropping an earlier pack's undelivered mail, the first mail states the whole current state, clears included
+    const mail = this.droppedOutbox && to.length ? composeState({ machine: this.machine, raised: alarms }) : compose({ machine: this.machine, transitions, reminders, raised: alarms });
+    if (this.droppedOutbox && to.length) this.droppedOutbox = 0;
     if (mail) {
       for (const a of alarms) this.state[a.key].lastMailAt = now;
-      for (const address of to) this.outbox.enqueue(address, `health-${this.machine}-${now}-${address.split(':').at(-1).slice(0, 12)}`, mail.subject, mail.body);
       this.log(`${mail.subject}${to.length ? '' : ' (no maintainer subscribed)'}\n${mail.body}`);
     }
-    const delivery = this.outbox.items.length ? await this.outbox.flush() : { pending: 0, error: this.outbox.error };
     const snapshot = {
       version: 1, machine: this.machine, platform: this.platform, at: new Date(now).toISOString(),
       probe: { pid: process.pid, intervalSec: this.config.intervalSec, slowIntervalSec: this.config.slowIntervalSec, sampleMs: this.sampleMs, user: this.env.USER ?? null, root: this.reader.isRoot?.() ?? false, config: this.config.source },
@@ -120,10 +142,18 @@ export class Probe {
       alarms,
       transitions: transitions.map((t) => ({ key: t.key, from: t.from, to: t.to, summary: t.summary })),
       maintainers: to,
-      mail: delivery,
+      mail: this.lastMail ?? null,
       readings,
     };
+    // The reading is written first; the mail is filed after it, and what the filing did is written beside it.
     writeAtomic(this.f.latest, snapshot);
+    if (mail && to.length) {
+      const result = fileAlarm({ machine: this.machine, env: this.env, to, key: `health-${this.machine}-${now}`, subject: mail.subject, body: mail.body });
+      this.lastMail = { at: new Date(now).toISOString(), filed: result.filed.length, to: result.filed.map((f) => f.to), error: result.error };
+      if (result.error) this.log(result.error);
+      snapshot.mail = this.lastMail;
+      writeAtomic(this.f.latest, snapshot);
+    }
     if (now - this.lastHistory >= 60_000) { this.#history(snapshot, now); this.lastHistory = now; }
     return snapshot;
   }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@volter/supercode-health",
-  "version": "0.1.2",
+  "version": "0.1.4",
   "type": "module",
   "description": "Optional machine pack: a cheap, deterministic probe of a machine's vitals that mails the machine's maintainer when a threshold is crossed. Each feature sits behind its own permission barrier.",
   "license": "MIT",