@volter/supercode-health 0.1.7 → 0.1.8

package/README.md

@@ -42,6 +42,22 @@ still-raised alarm mails again every `repeatMin` (60). Mail comes from
 it, within its bounds, and `supercode message waiting` / `supercode message expired` list
 what it has not delivered.
 
+## Failures that keep happening
+
+The `supercode` feature also reads the machine's own records of failure each minute: the slow log
+(`$SUPERCODE_HOME/logs/slow*.jsonl`, every command and door whose outcome was not `ok`) and the mailbox's
+expiries (`supercode message expired --json`). A kind is one failure as it repeats (the same command or door
+with the same outcome, the same expiry reason, ids and numbers taken out). A kind seen `warn` (3) times in
+`windowMin` (60) raises `supercode.failing:<kind>`; it clears when the window holds none of it.
+
+`failures.json` in the health directory is the machine's ledger: the releases it ran, each from when it was first
+seen running, and when each kind was first seen and under which release. The ledger's first run records what is
+already failing as seeded, naming no release for it. A kind first seen under a release the ledger saw the machine
+change to, after that change, is new with that release, and its alarm names that release's pull requests (the
+release writes them into `@volter/supercode`'s `lib/release-changes.json`). With `"failures": { "sayOnChanges":
+true }` in the config, the pack also comments such a kind once on each of those pull requests through `gh`, as the
+machine's user; it is off by default.
+
 ## Config
 
 The config is `config.json` in the health directory. The machine's maintainer owns it

package/lib/failures.mjs

@@ -0,0 +1,165 @@
+// The machine's own records of failure, read as alarms (t_b3844842): what supercode already writes when something fails
+// and nobody is watching. Two records, both kept by every machine:
+// - the slow log (`<supercode home>/logs/slow*.jsonl`): every command and door, its outcome; one whose outcome is not
+//   `ok` (an exit code, a timeout, an error) failed;
+// - the mailbox's expiries (`supercode message expired --json`): mail whose delivery ran out of its bound, and why.
+// A kind is one failure as it repeats: the same command or door with the same outcome, or the same expiry reason, with
+// its ids, numbers and paths taken out. A kind that keeps happening in the window is an alarm, named with the release
+// the machine runs, the first release this machine showed it under (kept in the ledger), and the pull requests that
+// release carries (its `lib/release-changes.json`, written by the release), so it lands on the changes that caused it.
+import { createHash } from 'node:crypto';
+import { existsSync, readdirSync, readFileSync, renameSync, statSync, writeFileSync } from 'node:fs';
+import { homedir } from 'node:os';
+import { dirname, join, resolve } from 'node:path';
+import { fileURLToPath } from 'node:url';
+
+/** A failure's text with what differs between its repeats taken out: ids, hashes, numbers, paths, sockets. */
+export function kindOf(text) {
+  return String(text ?? '')
+    .replace(/\b[mqt]-[0-9a-f]{6,}\b/g, '<id>')
+    .replace(/\b[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}\b/gi, '<uuid>')
+    .replace(/\bsc:[^\s"',)]+/g, '<address>')
+    .replace(/(uds:)?\/[^\s"',)]*/g, '<path>')
+    .replace(/\b\d+(\.\d+)?\b/g, '<n>')
+    .replace(/\s+/g, ' ')
+    .trim()
+    .slice(0, 160);
+}
+
+const keyOf = (kind) => createHash('sha256').update(kind).digest('hex').slice(0, 12);
+
+/** The slow log's failures since `since`: kind → { count, last }. Throws when the log's folder cannot be read. */
+export function slowFailures(env, since) {
+  const home = env.SUPERCODE_HOME || join(env.XDG_CONFIG_HOME || join(homedir(), '.config'), 'supercode');
+  const dir = join(home, 'logs');
+  const kinds = new Map();
+  if (!existsSync(dir)) return kinds; // nothing slow or failed was ever logged here
+  for (const name of readdirSync(dir)) {
+    if (!/^slow.*\.jsonl$/.test(name)) continue;
+    const file = join(dir, name);
+    if (statSync(file).mtimeMs < since) continue; // a rotation older than the window holds nothing in it
+    for (const line of readFileSync(file, 'utf8').split('\n')) {
+      if (!line) continue;
+      let row = null; try { row = JSON.parse(line); } catch { continue; }
+      const at = Date.parse(row?.at);
+      // `running` is a progress line (a command still running at 5 s says what it waits in), not an outcome
+      if (!(at >= since) || !row.outcome || row.outcome === 'ok' || row.outcome === 'running') continue;
+      const error = row.error ?? row.extra?.error ?? null;
+      const kind = kindOf(`${row.kind ?? 'call'} ${row.name ?? '?'}: ${row.outcome}${error ? `: ${error}` : ''}`);
+      const seen = kinds.get(kind) ?? { count: 0, last: 0, source: 'slow log' };
+      seen.count += 1; seen.last = Math.max(seen.last, at);
+      kinds.set(kind, seen);
+    }
+  }
+  return kinds;
+}
+
+/** The mailbox's expiries since `since` (rows of `supercode message expired --json`): kind → { count, last }. */
+export function expiredFailures(rows, since) {
+  const kinds = new Map();
+  for (const row of rows ?? []) {
+    const at = Number(row?.expired_at_ms);
+    if (!(at >= since)) continue;
+    const kind = kindOf(`mail expired undelivered: ${row.reason ?? 'no reason given'}`);
+    const seen = kinds.get(kind) ?? { count: 0, last: 0, source: 'mailbox expiries' };
+    seen.count += 1; seen.last = Math.max(seen.last, at);
+    kinds.set(kind, seen);
+  }
+  return kinds;
+}
+
+/**
+ * The release this pack runs in: the @volter/supercode package it is installed inside (one release is one install,
+ * #1187), its version, and the changes its release recorded. Null outside a release (a checkout).
+ */
+export function runningRelease(from = fileURLToPath(import.meta.url)) {
+  // <release>/node_modules/@volter/supercode-health/lib/failures.mjs
+  const pkg = resolve(dirname(from), '..', '..', '..', '..');
+  let manifest = null; try { manifest = JSON.parse(readFileSync(join(pkg, 'package.json'), 'utf8')); } catch { return null; }
+  if (manifest?.name !== '@volter/supercode') return null;
+  let changes = null; try { changes = JSON.parse(readFileSync(join(pkg, 'lib', 'release-changes.json'), 'utf8')); } catch { /* a release from before it was written */ }
+  return { version: manifest.version, changes };
+}
+
+/**
+ * What this machine has seen: the releases it ran, in order, each from when it was first seen running, and each kind,
+ * when and under which release it was first seen. The ledger's first run records what is already failing as `seeded`:
+ * those were here before anything watched, so no release is named for them as their cause.
+ */
+export class Ledger {
+  constructor(file) {
+    this.file = file;
+    let kept = null; try { kept = JSON.parse(readFileSync(file, 'utf8')); } catch { /* the first run on this machine */ }
+    this.fresh = !kept;
+    this.kinds = kept?.kinds ?? {};
+    this.releases = kept?.releases ?? [];
+  }
+  /** The release running now, recorded when it is not the last one recorded. */
+  running(release, at) {
+    const version = release?.version ?? null;
+    if (version && this.releases.at(-1)?.version !== version) { this.releases.push({ version, since: new Date(at).toISOString() }); this.save(); }
+  }
+  seen(kind, at, release) {
+    const key = keyOf(kind);
+    if (!this.kinds[key]) { this.kinds[key] = { kind, firstAt: new Date(at).toISOString(), firstRelease: release?.version ?? null, changes: release?.changes ?? null, seeded: this.fresh, said: [] }; this.save(); }
+    return this.kinds[key];
+  }
+  /** The first cycle is over: kinds seen from now on are new. */
+  settled() { if (this.fresh) { this.fresh = false; this.save(); } }
+  /**
+   * Whether a kind is new with its release: not seeded, first seen under a release this machine was seen to change to
+   * (one recorded after another), after that change, and so absent under the release before it.
+   */
+  newWith(first) {
+    if (first.seeded || !first.firstRelease) return false;
+    const index = this.releases.findIndex((r) => r.version === first.firstRelease);
+    return index > 0 && Date.parse(first.firstAt) >= Date.parse(this.releases[index].since);
+  }
+  said(kind, where) { const k = this.kinds[keyOf(kind)]; if (k && !k.said.includes(where)) { k.said.push(where); this.save(); } }
+  save() { const temp = `${this.file}.${process.pid}.tmp`; writeFileSync(temp, JSON.stringify({ releases: this.releases, kinds: this.kinds }, null, 2) + '\n', { mode: 0o600 }); renameSync(temp, this.file); }
+}
+
+/** The alarm's words: the kind, how often, the release it runs and the release that first showed it, with its changes. */
+export function failureSummary({ kind, count, windowMin, source }, first, release, isNew) {
+  const prs = first.changes?.prs ?? [];
+  // a release is named as the cause only when the kind is new with it (Ledger.newWith)
+  const since = first.seeded ? `already happening when this machine's records were first read (${first.firstAt})`
+    : isNew ? `new with supercode ${first.firstRelease} (first ${first.firstAt}, absent under the release before)${prs.length ? `; its changes: ${prs.map((pr) => `#${pr.number}`).join(', ')}` : ''}`
+    : `first ${first.firstAt} under supercode ${first.firstRelease ?? 'unknown'}, with no release change seen before it`;
+  return `${kind} — ${count} in ${windowMin} min (${source}); ${since}; running ${release?.version ?? 'unknown'}`;
+}
+
+export const keyFor = (kind) => `supercode.failing:${keyOf(kind)}`;
+
+/**
+ * Each failing kind new with a release (Ledger.newWith) said once on that release's changes: a comment on every pull
+ * request it carries (its `release-changes.json`), through `gh` as this machine's user. Once per kind and release; a
+ * try that failed is tried again an hour later. Off unless `failures.sayOnChanges: true` is in the config.
+ */
+export async function sayOnChanges({ failing, machine, release, ledger, env, config, gh = 'gh' }) {
+  // off unless the config turns it on (`failures.sayOnChanges: true`): only a kind new with a release is said, and on
+  // a machine whose ledger has seen a release change
+  if (config.failures?.sayOnChanges !== true) return;
+  const { run } = await import('./run.mjs');
+  for (const { kind, count, first } of failing) {
+    if (!ledger.newWith(first)) continue;
+    const prs = first.changes?.prs ?? [];
+    const repo = first.changes?.repository;
+    if (!prs.length || !repo) continue;
+    const where = `changes:${first.firstRelease}`;
+    if (first.said.includes(where)) continue;
+    if (first.triedAt && Date.now() - Date.parse(first.triedAt) < 3_600_000) continue;
+    const body = [
+      `**supercode on ${machine}: a failure keeps happening** — \`${kind}\`, ${count} times in the last hour.`,
+      `It is new with supercode ${first.firstRelease} on this machine (first ${first.firstAt}, absent under the release before it), the release this pull request is in with ${prs.map((pr) => `#${pr.number}`).join(', ')}; it runs ${release?.version ?? 'an unknown release'} now.`,
+      'Said by the machine-health pack from the machine\'s own records (sdk/health/lib/failures.mjs); its maintainer has the alarm too.',
+    ].join('\n\n');
+    let all = true;
+    for (const pr of prs) {
+      const posted = await run(gh, ['pr', 'comment', String(pr.number), '--repo', repo, '--body', body], { timeoutMs: 20_000, env });
+      if (!posted.ok) all = false;
+    }
+    if (all) ledger.said(kind, where);
+    else { first.triedAt = new Date().toISOString(); ledger.save(); }
+  }
+}

package/lib/sample.mjs

@@ -4,6 +4,9 @@ import { cpus, homedir, loadavg } from 'node:os';
 import { existsSync, readFileSync, statSync } from 'node:fs';
 import { run } from './run.mjs';
 import { rendererOrigins, originOf, localPort } from './chrome.mjs';
+import { expiredFailures, failureSummary, keyFor, Ledger, runningRelease, sayOnChanges, slowFailures } from './failures.mjs';
+import { healthDir } from './paths.mjs';
+import { join } from 'node:path';
 
 const MB = 1024 * 1024;
 const GB = 1024 * MB;
@@ -415,11 +418,12 @@ export class Sampler {
       // enrollment record, whose connection field is not kept current.
       // Liveness is the daemon's cheapest door (`describe`); `launches.list` walks every pane and can be slow
       // on a busy machine without the daemon being down, so its timeout is a reading, not an alarm.
-      const [status, describe, launches, machines] = await Promise.all([
+      const [status, describe, launches, machines, expired] = await Promise.all([
         run(bin, ['teams', 'status', '--json'], { timeoutMs: 20_000, env: this.env }),
         run(bin, ['teams', 'describe'], { timeoutMs: 15_000, env: this.env }).then((r) => ({ ...r, ms: Date.now() - t0 })),
         run(bin, ['teams', 'launches', 'list'], { timeoutMs: 45_000, env: this.env }),
         run(bin, ['teams', 'machines', 'list', '--json'], { timeoutMs: 20_000, env: this.env }),
+        run(bin, ['message', 'expired', '--json'], { timeoutMs: 20_000, env: this.env }),
       ]);
       let described = null; try { described = JSON.parse(describe.stdout); } catch { /* the daemon did not answer */ }
       let data = null; try { data = JSON.parse(status.stdout).data; } catch { /* unreadable */ }
@@ -454,6 +458,33 @@ export class Sampler {
       if (Array.isArray(rows)) slow.observations.push({ covered: 'supercode.launch:', family: 'supercode', all: true }, { covered: 'supercode.launchFailed:', family: 'supercode', all: true });
       for (const r of stuck) slow.observations.push({ key: `supercode.launch:${r.key}`, family: 'supercode', def: { forSec: 0 }, value: 1, met: { warn: true, crit: false, clears: false }, summary: r.status === 'issued' ? `launch ${r.key} (${r.kind ?? 'pane'}) issued ${Math.round((now - r.at) / 60_000)} min ago and never became ready` : `launch ${r.key} is live but its first input was not delivered: ${r.inputError}` });
       for (const r of failed) slow.observations.push({ key: `supercode.launchFailed:${r.key}`, family: 'supercode', def: { forSec: 0 }, value: 1, met: { warn: true, crit: false, clears: false }, summary: `launch ${r.key} (${r.kind ?? 'pane'}) failed at ${new Date(r.at).toISOString()}` });
+
+      // the machine's own records of failure (t_b3844842): each kind that keeps happening is an alarm naming the release
+      // that first showed it here and that release's changes; a record that could not be read leaves its kinds unknown
+      const fdef = alarm('supercode.failing');
+      const since = now - fdef.windowMin * 60_000;
+      const kinds = new Map();
+      const unread = [];
+      try { for (const [kind, seen] of slowFailures(this.env, since)) kinds.set(kind, seen); } catch (error) { unread.push(`the slow log could not be read: ${error.message}`); }
+      let expiries = null; try { expiries = JSON.parse(expired.stdout); } catch { unread.push(`the mailbox's expiries could not be listed: ${expired.error || 'no answer'}`); }
+      if (Array.isArray(expiries)) for (const [kind, seen] of expiredFailures(expiries, since)) kinds.set(kind, seen);
+      if (unread.length) slow.observations.push({ unreadPrefix: 'supercode.failing:', family: 'supercode', why: unread.join('; ') });
+      else slow.observations.push({ covered: 'supercode.failing:', family: 'supercode', all: true });
+      // read again each cycle: an upgrade replaces this pack's files under a probe that keeps running
+      const release = runningRelease();
+      this.ledger ??= new Ledger(join(healthDir(this.env), 'failures.json'));
+      this.ledger.running(release, now);
+      const failing = [];
+      for (const [kind, seen] of kinds) {
+        const first = this.ledger.seen(kind, seen.last, release);
+        const isNew = this.ledger.newWith(first);
+        const summary = failureSummary({ kind, count: seen.count, windowMin: fdef.windowMin, source: seen.source }, first, release, isNew);
+        slow.observations.push({ key: keyFor(kind), family: 'supercode', def: fdef, value: seen.count, unit: `in ${fdef.windowMin} min`, summary, detail: { kind, firstRelease: first.firstRelease, seeded: first.seeded, newWithRelease: isNew, running: release?.version ?? null, prs: isNew ? first.changes?.prs ?? [] : [] } });
+        if (seen.count >= fdef.warn) failing.push({ kind, count: seen.count, first, summary });
+      }
+      this.ledger.settled();
+      slow.readings.supercode.failing = failing.map((f) => ({ kind: f.kind, count: f.count, firstRelease: f.first.firstRelease }));
+      await sayOnChanges({ failing, machine: this.machine, release, ledger: this.ledger, env: this.env, config: cfg });
     });
 
     if (on.has('browser-tabs')) await attempt('browser-tabs', async () => {

package/lib/thresholds.mjs

@@ -34,6 +34,9 @@ export const DEFAULTS = {
     'service.down': { forSec: 60 },
     'supercode.connector': { forSec: 60 },
     'supercode.launch': { stuckMin: 5, failedWithinMin: 30 },
+    // one failure that keeps happening (the same command or door with the same outcome, the same mail expiry) in the
+    // window; it clears when the window holds none of it (t_b3844842)
+    'supercode.failing': { warn: 3, crit: 20, clear: 0, windowMin: 60, forSec: 0 },
     'browser.tabs': { warn: null },
     'power.thermal': { forSec: 60 },
   },

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@volter/supercode-health",
-  "version": "0.1.7",
+  "version": "0.1.8",
   "type": "module",
   "description": "Optional machine pack: a cheap, deterministic probe of a machine's vitals that mails the machine's maintainer when a threshold is crossed. Each feature sits behind its own permission barrier.",
   "license": "MIT",