@volter/supercode-health 0.1.13 → 0.1.15

package/lib/drift.mjs

@@ -71,7 +71,18 @@ export async function staleSupercode({ rows = null } = {}) {
     } else if (Number.isNaN(started) || installed <= started + 1000) continue; // ps says a start to the second
     stale.push({ pid: Number(pid), command: command.slice(0, 200), file, startedAt: new Date(started).toISOString(), installedAt: new Date(installed).toISOString() });
   }
-  return stale.sort((a, b) => b.installedAt.localeCompare(a.installedAt));
+  return moved(stale).sort((a, b) => b.installedAt.localeCompare(a.installedAt));
+}
+
+/** Less the processes that answer through the release on disk: an MCP server's pump says so in `<home>/run/mcp/<pid>`
+ *  once a child of the release an install put there answers for it (lib/mcp.mjs), at or after that install. */
+function moved(stale) {
+  const home = process.env.SUPERCODE_HOME || (process.env.XDG_CONFIG_HOME ? join(process.env.XDG_CONFIG_HOME, 'supercode') : join(homedir(), '.config', 'supercode'));
+  return stale.filter((row) => {
+    let said = null;
+    try { said = JSON.parse(readFileSync(join(home, 'run', 'mcp', String(row.pid)), 'utf8')); } catch { return true; }
+    return !(said?.pid === row.pid && Date.parse(said.at) >= Date.parse(row.installedAt));
+  });
 }
 
 /**

package/lib/features.mjs

@@ -13,6 +13,7 @@ export const FEATURES = [
   { id: 'disk-io', barrier: 'none', platforms: ['darwin', 'linux'], reads: 'device throughput; top writers per process' },
   { id: 'tcp', barrier: 'none', platforms: ['darwin', 'linux', 'win32'], reads: 'sockets per state, ephemeral ports in use, TIME_WAIT by listener and its pid' },
   { id: 'files', barrier: 'none', platforms: ['darwin', 'linux'], reads: 'open files against the system limit' },
+  { id: 'recorded', barrier: 'none', platforms: ['darwin', 'linux'], reads: 'every command supercode recorded outside itself (harness configs, hooks baked into live sessions), one of each program and verb run as its consumer runs it' },
   { id: 'drift', barrier: 'none', platforms: ['darwin', 'linux'], reads: 'running supercode processes whose install was replaced after they started (one ps)' },
   { id: 'services', barrier: 'none', platforms: ['darwin', 'linux'], reads: "configured launchd/systemd user jobs: running, last exit, log silence" },
   { id: 'supercode', barrier: 'integration:machine-daemon', platforms: ['darwin', 'linux', 'win32'], reads: "this machine's supercode connector: answering, server link, stuck launches" },

package/lib/probe.mjs

@@ -51,6 +51,9 @@ export class Probe {
     this.f = healthFiles(dir);
     this.machine = machineName(machine || env.SUPERCODE_HEALTH_MACHINE || hostname());
     this.state = {};
+    // what the last mail said of each alarm (key → its level, summary, detail): a mail says what changed since, never each
+    // flip in between (t_be880329: disk and memory flapping mailed the maintainer ~1,200 times a day on yuerans)
+    this.said = new Map(); this.changesMailAt = 0; this.raisedLastAt = new Map(); // key → when it was last seen raised
     this.unknowns = {}; this.unknownMailAt = {}; // unknownMailAt: a kind of unknown (unknownKind: its key's prefix) → last mailed
     this.lastHistory = 0;
     this.sampleMs = null;
@@ -99,6 +102,7 @@ export class Probe {
     // A raised alarm whose reading was unknown keeps its unknown since across the restart, as its unknowns entry does.
     for (const a of (previous?.alarms ?? []).filter((x) => !x.detail?.pid || alive(x.detail.pid))) this.state[a.key] = { key: a.key, family: a.family, level: a.level, warnSince: null, critSince: null, raisedAt: a.since ? Date.parse(a.since) : Date.now(), lastMailAt: this.droppedOutbox ? 0 : Date.now(), restoredUntil: Date.now() + 2 * 60_000, value: a.value, unit: a.unit, summary: a.summary, detail: a.detail,
       ...(a.unknown?.since ? { unread: a.unknown.why, unreadAt: Date.parse(a.unknown.since) } : {}) };
+    for (const a of (previous?.alarms ?? [])) this.said.set(a.key, { level: a.level, summary: a.summary, detail: a.detail ?? null });
   }
 
   /** Remove an earlier pack's outbox; answers how many mails it held, logged by recipient. */
@@ -138,7 +142,26 @@ export class Probe {
     const repeatMs = this.config.repeatMin * 60_000;
     // A raised alarm is reminded each repeatMin after it was last said: in a mail of its own transition or reminder, never
     // pushed back by a mail about something else.
-    const raisedNow = new Set(transitions.filter((t) => t.to !== 'none').map((t) => t.key));
+    // What changed since the last mail said it (net of every raise and clear between): said at most once each quietMin,
+    // a new crit at once, and with any mail going anyway (a reminder, an unknown due). A key that raised and cleared
+    // between two mails is never said.
+    const quietMs = (this.config.quietMin ?? 15) * 60_000;
+    const levels = new Map(alarms.map((a) => [a.key, a]));
+    const lastTransition = new Map(transitions.map((t) => [t.key, t]));
+    const net = [];
+    for (const a of alarms) {
+      const was = this.said.get(a.key)?.level ?? 'none';
+      if (was !== a.level) net.push({ key: a.key, from: was, to: a.level, value: a.value, unit: a.unit ?? null, summary: a.summary, detail: a.detail ?? null });
+    }
+    for (const a of alarms) this.raisedLastAt.set(a.key, now);
+    for (const [key, was] of this.said) {
+      if (levels.has(key)) continue;
+      // a clear is said once it has held a whole quiet window: an alarm that flaps is said raised once, never each clear
+      if (now - (this.raisedLastAt.get(key) ?? 0) < quietMs) continue;
+      const t = lastTransition.get(key);
+      net.push({ key, from: was.level, to: 'none', value: null, summary: t?.to === 'none' ? t.summary : `${was.summary} — cleared`, detail: t?.detail ?? was.detail });
+    }
+    const raisedNow = new Set(net.filter((t) => t.to !== 'none').map((t) => t.key));
     const reminders = alarms.filter((a) => !raisedNow.has(a.key) && now - (this.state[a.key].lastMailAt ?? 0) >= repeatMs);
     // A reading unknown for unknownMin is the maintainers' to know (raised or not), and again each repeatMin until read.
     // Timed per kind (a key's prefix; a family's whole key): a kind is due once its earliest key has been unknown
@@ -159,14 +182,25 @@ export class Probe {
     const to = subscribers(this.f, this.config);
     // after dropping an earlier pack's undelivered mail, the first mail states the whole current state, clears included
     const composedState = Boolean(this.droppedOutbox && to.length);
-    const mail = composedState ? composeState({ machine: this.machine, raised: alarms }) : compose({ machine: this.machine, transitions, reminders, raised: alarms, unknowns: unknownsDue });
+    const changesDue = net.length > 0 && (now - this.changesMailAt >= quietMs || net.some((t) => t.to === 'crit' && t.from !== 'crit') || reminders.length > 0 || unknownsDue.length > 0);
+    const changes = changesDue ? net : [];
+    const mail = composedState ? composeState({ machine: this.machine, raised: alarms }) : compose({ machine: this.machine, transitions: changes, reminders: reminders.filter((a) => !changes.some((t) => t.key === a.key)), raised: alarms, unknowns: unknownsDue });
     if (composedState) this.droppedOutbox = 0;
     if (mail) {
       // the kinds of unknown this mail said (the whole-state mail says none: they stay due)
       if (!composedState) for (const [kind] of dueKinds) this.unknownMailAt[kind] = now;
       // only the alarms this mail said: raised by it, or reminded (the whole-state mail says every one)
-      const said = composedState ? alarms : alarms.filter((a) => raisedNow.has(a.key) || reminders.includes(a));
+      const said = composedState ? alarms : alarms.filter((a) => (changes.length && raisedNow.has(a.key)) || reminders.includes(a));
       for (const a of said) this.state[a.key].lastMailAt = now;
+      // the state this mail said is what the next mail is measured from: the whole state, or the last one said with this
+      // mail's changes applied. A clear still in its hold was not said, so its alarm stays said raised until a mail
+      // says the clear (#1268 review P1: replacing it with the alarms raised now lost that clear for good)
+      if (composedState) this.said = new Map(alarms.map((a) => [a.key, { level: a.level, summary: a.summary, detail: a.detail ?? null }]));
+      else for (const t of changes) {
+        if (t.to === 'none') this.said.delete(t.key);
+        else this.said.set(t.key, { level: t.to, summary: t.summary, detail: t.detail ?? null });
+      }
+      if (composedState || changes.length) this.changesMailAt = now;
       this.log(`${mail.subject}${to.length ? '' : ' (no maintainer subscribed)'}\n${mail.body}`);
     }
     const snapshot = {

package/lib/recorded.mjs

@@ -0,0 +1,136 @@
+// Recorded commands (t_4e879316, ADR 0022): every command this machine holds for supercode outside supercode itself
+// still runs. Supercode writes commands into things it does not own: a harness's config (Codex's hooks.json and
+// config.toml, Claude's settings and .claude.json) and the hooks baked into a session's launch flags, which the session
+// holds until it ends. A release that moved a verb out of the native left those answering `unrecognized subcommand`
+// (2026-10-06: Codex failed every tool call, Claude's AskUserQuestion was blocked). This reads them all, groups them by
+// what they run (program and verb), and runs one of each as its consumer runs it: a hook with {} on stdin, an MCP server
+// sent initialize. A hook may refuse by its own logic; what is said is a command that cannot run at all.
+import { mkdirSync, mkdtempSync, readFileSync, rmSync } from 'node:fs';
+import { spawn } from 'node:child_process';
+import { homedir, tmpdir } from 'node:os';
+import { join } from 'node:path';
+import { run } from './run.mjs';
+
+const json = (path) => { try { return JSON.parse(readFileSync(path, 'utf8')); } catch { return null; } };
+const text = (path) => { try { return readFileSync(path, 'utf8'); } catch { return ''; } };
+const mentions = (command) => /supercode/.test(command);
+const unquote = (value) => { try { return JSON.parse(`"${value}"`); } catch { return value.replace(/\\"/g, '"'); } };
+/** The `command` values in a JSON or TOML text: a hook's, an MCP server's. */
+const commandsIn = (source) => [...source.matchAll(/"command"\s*[:=]\s*"((?:\\.|[^"\\])*)"|\bcommand\s*=\s*'([^']*)'/g)].map((m) => (m[1] !== undefined ? unquote(m[1]) : m[2]));
+
+/** What a command runs, for grouping: its program (the first path in it) and the supercode verb after it. */
+function shape(command) {
+  const words = command.match(/'[^']*'|"[^"]*"|\S+/g)?.map((word) => word.replace(/^['"]|['"]$/g, '')) ?? [];
+  const at = words.findIndex((word) => /supercode/.test(word) && !word.includes('='));
+  if (at < 0) return null;
+  return `${words[at]} ${words.slice(at + 1).filter((word) => !word.startsWith('/') && !word.startsWith("'")).slice(0, 2).join(' ')}`.trim();
+}
+
+/** Every recorded command naming supercode: `{ command, kind: 'hook' | 'mcp', where }`. */
+export async function recordedCommands({ home = homedir(), processes = true } = {}) {
+  const found = [];
+  const hooks = (config, where) => {
+    for (const [event, groups] of Object.entries(config?.hooks ?? {})) for (const group of groups ?? []) for (const hook of group.hooks ?? []) {
+      if (typeof hook.command === 'string' && mentions(hook.command)) found.push({ command: hook.command, kind: 'hook', where: `${where} ${event}` });
+    }
+  };
+  hooks(json(join(home, '.codex', 'hooks.json')), 'codex hooks.json');
+  for (const file of ['settings.json', 'settings.local.json']) hooks(json(join(home, '.claude', file)), `claude ${file}`);
+  for (const [name, server] of Object.entries(json(join(home, '.claude.json'))?.mcpServers ?? {})) {
+    const command = [server.command, ...(server.args ?? [])].map((word) => `'${String(word).replace(/'/g, `'\\''`)}'`).join(' ');
+    if (mentions(command)) found.push({ command, kind: 'mcp', where: `claude mcp ${name}` });
+  }
+  // a section runs to the next header line (`[name]` opening a line), not to the next `[` (an `args = [...]` array)
+  for (const m of text(join(home, '.codex', 'config.toml')).matchAll(/^\[mcp_servers\.([^\]\n]+)\]\s*\n((?:(?!\[)[^\n]*\n?)*)/gm)) {
+    const program = /command\s*=\s*"([^"]+)"/.exec(m[2])?.[1];
+    const args = [...(/args\s*=\s*\[([^\]]*)\]/.exec(m[2])?.[1] ?? '').matchAll(/"([^"]*)"/g)].map((a) => a[1]);
+    const command = [program, ...args].map((word) => `'${String(word).replace(/'/g, `'\\''`)}'`).join(' ');
+    if (program && mentions(command)) found.push({ command, kind: 'mcp', where: `codex mcp ${m[1]}` });
+  }
+  // the hooks baked into running sessions: Claude's --settings (a file, or inline JSON) and Codex's -c values
+  if (processes) {
+    const table = String((await run('ps', ['-A', '-ww', '-o', 'pid=,command='], { timeoutMs: 10_000 })).stdout ?? '');
+    for (const line of table.split('\n')) {
+      const m = /^\s*(\d+)\s+\S*?(?:^|\/)(claude|codex)(\s.*)$/.exec(line);
+      if (!m) continue;
+      const sources = [m[3], ...[...m[3].matchAll(/--settings\s+(\/\S+\.json)/g)].map((f) => text(f[1]))];
+      for (const source of sources) for (const command of commandsIn(source)) if (mentions(command)) found.push({ command, kind: 'hook', where: `${m[2]} session ${m[1]}` });
+    }
+  }
+  return found;
+}
+
+/**
+ * The command as it can be run with no effect on this machine's live state (#1255 review P1: run as recorded, a relay's
+ * gate settled its queued send and relay-inbound overwrote its turn): its program and verb exactly as recorded, every
+ * data path after them (a folder, a binding; never a script an interpreter runs) a path that does not exist inside
+ * `root`, and no leading `env NAME=…` (which would name the live
+ * home again). Run under `root`'s own HOME and supercode homes, it reaches no live daemon, relay, binding or mailbox:
+ * what it shows is whether its program runs that verb at all.
+ */
+export function isolated(command, root) {
+  const words = command.match(/'(?:[^']|'\\'')*'|"(?:\\.|[^"\\])*"|\S+/g) ?? [];
+  const bare = (word) => word.replace(/^'(.*)'$/s, '$1').replace(/^"(.*)"$/s, '$1');
+  let at = 0;
+  if (bare(words[0] ?? '') === 'env') { at = 1; while (at < words.length && /^[A-Za-z_][A-Za-z0-9_]*=/.test(bare(words[at]))) at += 1; }
+  const [program, ...rest] = words.slice(at);
+  if (!program) return null;
+  const quote = (word) => `'${word.replace(/'/g, `'\\''`)}'`;
+  // a script an interpreter runs (`node …/guard.mjs`) is what runs, kept; a data path (a relay's folder, a binding) is not
+  const data = (word) => word.startsWith('/') && !/\.(?:m?js|cjs)$/.test(word);
+  return [quote(bare(program)), ...rest.map((word, index) => (data(bare(word)) ? quote(join(root, 'absent', String(index))) : quote(bare(word))))].join(' ');
+}
+
+/** The environment a probe runs in: `root`'s own homes, nothing of this machine's. */
+function isolatedEnv(root) {
+  for (const dir of ['home', 'supercode', 'xdg', 'tmp']) mkdirSync(join(root, dir), { recursive: true });
+  return { PATH: process.env.PATH ?? '/usr/bin:/bin', HOME: join(root, 'home'), SUPERCODE_HOME: join(root, 'supercode'), XDG_CONFIG_HOME: join(root, 'xdg'), TMPDIR: join(root, 'tmp'), CODEX_HOME: join(root, 'home', '.codex'), CLAUDE_CONFIG_DIR: join(root, 'home', '.claude') };
+}
+
+/** Run `command` as its consumer would, isolated (above); `{ code, stdout, stderr }`. */
+function attempt(command, { mcp, root }) {
+  return new Promise((done) => {
+    const safe = isolated(command, root);
+    if (!safe) { done({ code: 127, stdout: '', stderr: 'no program in the command' }); return; }
+    const child = spawn('/bin/sh', ['-c', safe], { stdio: ['pipe', 'pipe', 'pipe'], env: isolatedEnv(root), cwd: root });
+    let stdout = '', stderr = '', settled = false;
+    const finish = (answer) => { if (settled) return; settled = true; clearTimeout(timer); try { child.kill('SIGKILL'); } catch { /* gone */ } done(answer); };
+    const timer = setTimeout(() => finish({ code: null, stdout, stderr: `${stderr}\n(no answer within 15 s)` }), 15_000);
+    child.on('error', (error) => finish({ code: 127, stdout, stderr: error.message }));
+    child.stdout.on('data', (data) => { stdout += data; if (mcp && /"result"/.test(stdout)) finish({ code: 0, stdout, stderr }); });
+    child.stderr.on('data', (data) => { stderr += data; });
+    child.on('close', (code) => finish({ code, stdout, stderr }));
+    child.stdin.end(mcp ? `${JSON.stringify({ jsonrpc: '2.0', id: 1, method: 'initialize', params: { protocolVersion: '2025-06-18', capabilities: {}, clientInfo: { name: 'supercode-health', version: '1' } } })}\n` : '{}');
+  });
+}
+
+const cannotRun = ({ code, stderr }) => code === null || code === 126 || code === 127
+  || /unrecognized subcommand|unexpected argument|command not found|No such file|Cannot find module|Exec format/i.test(stderr);
+
+/**
+ * The recorded commands that cannot run, one per shape (program and verb):
+ * `[{ shape, command, kind, where: [..], count, code, error }]`. `commands` (for a reading without the machine's) are
+ * `recordedCommands()`'s.
+ */
+export async function brokenRecorded({ commands = null } = {}) {
+  const all = commands ?? (await recordedCommands());
+  const groups = new Map();
+  for (const entry of all) {
+    const key = `${entry.kind} ${shape(entry.command) ?? entry.command}`;
+    const group = groups.get(key) ?? { key, ...entry, where: [], count: 0 };
+    group.where.push(entry.where);
+    group.count += 1;
+    groups.set(key, group);
+  }
+  const broken = [];
+  const root = mkdtempSync(join(tmpdir(), 'supercode-recorded-'));
+  try {
+  await Promise.all([...groups.values()].map(async (group) => {
+    const answer = await attempt(group.command, { mcp: group.kind === 'mcp', root });
+    const failed = group.kind === 'mcp' ? !/"result"/.test(answer.stdout) : cannotRun(answer);
+    if (failed) broken.push({ shape: group.key, command: group.command.slice(0, 300), kind: group.kind, where: [...new Set(group.where)].slice(0, 10), count: group.count, code: answer.code, error: answer.stderr.trim().split('\n')[0]?.slice(0, 200) ?? '' });
+  }));
+  } finally { rmSync(root, { recursive: true, force: true }); }
+  return broken.sort((a, b) => b.count - a.count);
+}
+

package/lib/sample.mjs

@@ -8,6 +8,7 @@ import { rendererOrigins, originOf, localPort } from './chrome.mjs';
 import { expiredFailures, failureSummary, keyFor, Ledger, runningRelease, sayOnChanges, slowFailures } from './failures.mjs';
 import { healthDir } from './paths.mjs';
 import { pinnedHomes, staleSupercode } from './drift.mjs';
+import { brokenRecorded } from './recorded.mjs';
 import { join } from 'node:path';
 
 const MB = 1024 * 1024;
@@ -399,6 +400,14 @@ export class Sampler {
       slow.readings.pinned = pinned;
       for (const p of pinned) slow.observations.push({ key: `release.pinned:${p.home}`, family: 'pinned', def: alarm('release.pinned'), value: 1, met: { warn: true, crit: false, clears: false }, summary: `board home ${p.home} is ${p.why} (orchestrator ${p.version ?? 'unknown'})`, detail: p });
     });
+    if (on.has('recorded')) await attempt('recorded', async () => {
+      const broken = await brokenRecorded();
+      slow.families.add('recorded');
+      // every broken shape this pass found; one that runs again is cleared
+      slow.observations.push({ enumerated: 'recorded', family: 'recorded', by: 'suffix', entities: new Set(broken.map((b) => b.shape)) });
+      slow.readings.recorded = broken;
+      for (const b of broken) slow.observations.push({ key: `recorded.broken:${b.shape}`, family: 'recorded', def: alarm('recorded.broken'), value: b.count, met: { warn: true, crit: true, clears: false }, summary: `${b.count} recorded command(s) cannot run (${b.where.slice(0, 3).join(', ')}): ${b.error}; ${b.command.slice(0, 120)}`, detail: b });
+    });
     if (on.has('services')) await attempt('services', async () => {
       const list = cfg.services ?? [];
       const jobs = await this.reader.services(list.map((s) => s.label));

package/lib/thresholds.mjs

@@ -5,6 +5,7 @@ export const DEFAULTS = {
   slowIntervalSec: 60,      // disk space, services, the connector, tabs
   footprintTopEverySec: 300, // macOS: every user's largest footprints through top (about 2 s of top per run)
   repeatMin: 60,            // a raised alarm mails again this often (netdata's repeat cap)
+  quietMin: 15,             // what changed is mailed at most this often (a new crit at once): a flapping alarm is no stream
   unknownMin: 15,           // a reading unknown this long (raised or not) is mailed to the maintainers, and again each repeatMin
   historyHours: 24,
   daemons: {
@@ -36,6 +37,8 @@ export const DEFAULTS = {
     'release.drift': { forSec: 300 },
     // a board home its record holds to another install's orchestrator: said once it has stayed so 5 min
     'release.pinned': { forSec: 300 },
+    // a recorded command that cannot run fails its harness on every use (a hook on every tool call): said at once
+    'recorded.broken': { forSec: 0 },
     'supercode.connector': { forSec: 60 },
     'supercode.launch': { stuckMin: 5, failedWithinMin: 30 },
     // one failure that keeps happening (the same command or door with the same outcome, the same mail expiry) in the

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@volter/supercode-health",
-  "version": "0.1.13",
+  "version": "0.1.15",
   "type": "module",
   "description": "Optional machine pack: a cheap, deterministic probe of a machine's vitals that mails the machine's maintainer when a threshold is crossed. Each feature sits behind its own permission barrier.",
   "license": "MIT",
@@ -9,7 +9,8 @@
     "./bin": "./bin/health.mjs",
     "./format": "./lib/format.mjs",
     "./paths": "./lib/paths.mjs",
-    "./drift": "./lib/drift.mjs"
+    "./drift": "./lib/drift.mjs",
+    "./recorded": "./lib/recorded.mjs"
   },
   "bin": {
     "supercode-health": "./bin/health.mjs"