@volter/supercode-health 0.1.6 → 0.1.8

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/README.md CHANGED
@@ -42,6 +42,22 @@ still-raised alarm mails again every `repeatMin` (60). Mail comes from
42
42
  it, within its bounds, and `supercode message waiting` / `supercode message expired` list
43
43
  what it has not delivered.
44
44
 
45
+ ## Failures that keep happening
46
+
47
+ The `supercode` feature also reads the machine's own records of failure each minute: the slow log
48
+ (`$SUPERCODE_HOME/logs/slow*.jsonl`, every command and door whose outcome was not `ok`) and the mailbox's
49
+ expiries (`supercode message expired --json`). A kind is one failure as it repeats (the same command or door
50
+ with the same outcome, the same expiry reason, ids and numbers taken out). A kind seen `warn` (3) times in
51
+ `windowMin` (60) raises `supercode.failing:<kind>`; it clears when the window holds none of it.
52
+
53
+ `failures.json` in the health directory is the machine's ledger: the releases it ran, each from when it was first
54
+ seen running, and when each kind was first seen and under which release. The ledger's first run records what is
55
+ already failing as seeded, naming no release for it. A kind first seen under a release the ledger saw the machine
56
+ change to, after that change, is new with that release, and its alarm names that release's pull requests (the
57
+ release writes them into `@volter/supercode`'s `lib/release-changes.json`). With `"failures": { "sayOnChanges":
58
+ true }` in the config, the pack also comments such a kind once on each of those pull requests through `gh`, as the
59
+ machine's user; it is off by default.
60
+
45
61
  ## Config
46
62
 
47
63
  The config is `config.json` in the health directory. The machine's maintainer owns it
@@ -0,0 +1,165 @@
1
+ // The machine's own records of failure, read as alarms (t_b3844842): what supercode already writes when something fails
2
+ // and nobody is watching. Two records, both kept by every machine:
3
+ // - the slow log (`<supercode home>/logs/slow*.jsonl`): every command and door, its outcome; one whose outcome is not
4
+ // `ok` (an exit code, a timeout, an error) failed;
5
+ // - the mailbox's expiries (`supercode message expired --json`): mail whose delivery ran out of its bound, and why.
6
+ // A kind is one failure as it repeats: the same command or door with the same outcome, or the same expiry reason, with
7
+ // its ids, numbers and paths taken out. A kind that keeps happening in the window is an alarm, named with the release
8
+ // the machine runs, the first release this machine showed it under (kept in the ledger), and the pull requests that
9
+ // release carries (its `lib/release-changes.json`, written by the release), so it lands on the changes that caused it.
10
+ import { createHash } from 'node:crypto';
11
+ import { existsSync, readdirSync, readFileSync, renameSync, statSync, writeFileSync } from 'node:fs';
12
+ import { homedir } from 'node:os';
13
+ import { dirname, join, resolve } from 'node:path';
14
+ import { fileURLToPath } from 'node:url';
15
+
16
+ /** A failure's text with what differs between its repeats taken out: ids, hashes, numbers, paths, sockets. */
17
+ export function kindOf(text) {
18
+ return String(text ?? '')
19
+ .replace(/\b[mqt]-[0-9a-f]{6,}\b/g, '<id>')
20
+ .replace(/\b[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}\b/gi, '<uuid>')
21
+ .replace(/\bsc:[^\s"',)]+/g, '<address>')
22
+ .replace(/(uds:)?\/[^\s"',)]*/g, '<path>')
23
+ .replace(/\b\d+(\.\d+)?\b/g, '<n>')
24
+ .replace(/\s+/g, ' ')
25
+ .trim()
26
+ .slice(0, 160);
27
+ }
28
+
29
+ const keyOf = (kind) => createHash('sha256').update(kind).digest('hex').slice(0, 12);
30
+
31
+ /** The slow log's failures since `since`: kind → { count, last }. Throws when the log's folder cannot be read. */
32
+ export function slowFailures(env, since) {
33
+ const home = env.SUPERCODE_HOME || join(env.XDG_CONFIG_HOME || join(homedir(), '.config'), 'supercode');
34
+ const dir = join(home, 'logs');
35
+ const kinds = new Map();
36
+ if (!existsSync(dir)) return kinds; // nothing slow or failed was ever logged here
37
+ for (const name of readdirSync(dir)) {
38
+ if (!/^slow.*\.jsonl$/.test(name)) continue;
39
+ const file = join(dir, name);
40
+ if (statSync(file).mtimeMs < since) continue; // a rotation older than the window holds nothing in it
41
+ for (const line of readFileSync(file, 'utf8').split('\n')) {
42
+ if (!line) continue;
43
+ let row = null; try { row = JSON.parse(line); } catch { continue; }
44
+ const at = Date.parse(row?.at);
45
+ // `running` is a progress line (a command still running at 5 s says what it waits in), not an outcome
46
+ if (!(at >= since) || !row.outcome || row.outcome === 'ok' || row.outcome === 'running') continue;
47
+ const error = row.error ?? row.extra?.error ?? null;
48
+ const kind = kindOf(`${row.kind ?? 'call'} ${row.name ?? '?'}: ${row.outcome}${error ? `: ${error}` : ''}`);
49
+ const seen = kinds.get(kind) ?? { count: 0, last: 0, source: 'slow log' };
50
+ seen.count += 1; seen.last = Math.max(seen.last, at);
51
+ kinds.set(kind, seen);
52
+ }
53
+ }
54
+ return kinds;
55
+ }
56
+
57
+ /** The mailbox's expiries since `since` (rows of `supercode message expired --json`): kind → { count, last }. */
58
+ export function expiredFailures(rows, since) {
59
+ const kinds = new Map();
60
+ for (const row of rows ?? []) {
61
+ const at = Number(row?.expired_at_ms);
62
+ if (!(at >= since)) continue;
63
+ const kind = kindOf(`mail expired undelivered: ${row.reason ?? 'no reason given'}`);
64
+ const seen = kinds.get(kind) ?? { count: 0, last: 0, source: 'mailbox expiries' };
65
+ seen.count += 1; seen.last = Math.max(seen.last, at);
66
+ kinds.set(kind, seen);
67
+ }
68
+ return kinds;
69
+ }
70
+
71
+ /**
72
+ * The release this pack runs in: the @volter/supercode package it is installed inside (one release is one install,
73
+ * #1187), its version, and the changes its release recorded. Null outside a release (a checkout).
74
+ */
75
+ export function runningRelease(from = fileURLToPath(import.meta.url)) {
76
+ // <release>/node_modules/@volter/supercode-health/lib/failures.mjs
77
+ const pkg = resolve(dirname(from), '..', '..', '..', '..');
78
+ let manifest = null; try { manifest = JSON.parse(readFileSync(join(pkg, 'package.json'), 'utf8')); } catch { return null; }
79
+ if (manifest?.name !== '@volter/supercode') return null;
80
+ let changes = null; try { changes = JSON.parse(readFileSync(join(pkg, 'lib', 'release-changes.json'), 'utf8')); } catch { /* a release from before it was written */ }
81
+ return { version: manifest.version, changes };
82
+ }
83
+
84
+ /**
85
+ * What this machine has seen: the releases it ran, in order, each from when it was first seen running, and each kind,
86
+ * when and under which release it was first seen. The ledger's first run records what is already failing as `seeded`:
87
+ * those were here before anything watched, so no release is named for them as their cause.
88
+ */
89
+ export class Ledger {
90
+ constructor(file) {
91
+ this.file = file;
92
+ let kept = null; try { kept = JSON.parse(readFileSync(file, 'utf8')); } catch { /* the first run on this machine */ }
93
+ this.fresh = !kept;
94
+ this.kinds = kept?.kinds ?? {};
95
+ this.releases = kept?.releases ?? [];
96
+ }
97
+ /** The release running now, recorded when it is not the last one recorded. */
98
+ running(release, at) {
99
+ const version = release?.version ?? null;
100
+ if (version && this.releases.at(-1)?.version !== version) { this.releases.push({ version, since: new Date(at).toISOString() }); this.save(); }
101
+ }
102
+ seen(kind, at, release) {
103
+ const key = keyOf(kind);
104
+ if (!this.kinds[key]) { this.kinds[key] = { kind, firstAt: new Date(at).toISOString(), firstRelease: release?.version ?? null, changes: release?.changes ?? null, seeded: this.fresh, said: [] }; this.save(); }
105
+ return this.kinds[key];
106
+ }
107
+ /** The first cycle is over: kinds seen from now on are new. */
108
+ settled() { if (this.fresh) { this.fresh = false; this.save(); } }
109
+ /**
110
+ * Whether a kind is new with its release: not seeded, first seen under a release this machine was seen to change to
111
+ * (one recorded after another), after that change, and so absent under the release before it.
112
+ */
113
+ newWith(first) {
114
+ if (first.seeded || !first.firstRelease) return false;
115
+ const index = this.releases.findIndex((r) => r.version === first.firstRelease);
116
+ return index > 0 && Date.parse(first.firstAt) >= Date.parse(this.releases[index].since);
117
+ }
118
+ said(kind, where) { const k = this.kinds[keyOf(kind)]; if (k && !k.said.includes(where)) { k.said.push(where); this.save(); } }
119
+ save() { const temp = `${this.file}.${process.pid}.tmp`; writeFileSync(temp, JSON.stringify({ releases: this.releases, kinds: this.kinds }, null, 2) + '\n', { mode: 0o600 }); renameSync(temp, this.file); }
120
+ }
121
+
122
+ /** The alarm's words: the kind, how often, the release it runs and the release that first showed it, with its changes. */
123
+ export function failureSummary({ kind, count, windowMin, source }, first, release, isNew) {
124
+ const prs = first.changes?.prs ?? [];
125
+ // a release is named as the cause only when the kind is new with it (Ledger.newWith)
126
+ const since = first.seeded ? `already happening when this machine's records were first read (${first.firstAt})`
127
+ : isNew ? `new with supercode ${first.firstRelease} (first ${first.firstAt}, absent under the release before)${prs.length ? `; its changes: ${prs.map((pr) => `#${pr.number}`).join(', ')}` : ''}`
128
+ : `first ${first.firstAt} under supercode ${first.firstRelease ?? 'unknown'}, with no release change seen before it`;
129
+ return `${kind} — ${count} in ${windowMin} min (${source}); ${since}; running ${release?.version ?? 'unknown'}`;
130
+ }
131
+
132
+ export const keyFor = (kind) => `supercode.failing:${keyOf(kind)}`;
133
+
134
+ /**
135
+ * Each failing kind new with a release (Ledger.newWith) said once on that release's changes: a comment on every pull
136
+ * request it carries (its `release-changes.json`), through `gh` as this machine's user. Once per kind and release; a
137
+ * try that failed is tried again an hour later. Off unless `failures.sayOnChanges: true` is in the config.
138
+ */
139
+ export async function sayOnChanges({ failing, machine, release, ledger, env, config, gh = 'gh' }) {
140
+ // off unless the config turns it on (`failures.sayOnChanges: true`): only a kind new with a release is said, and on
141
+ // a machine whose ledger has seen a release change
142
+ if (config.failures?.sayOnChanges !== true) return;
143
+ const { run } = await import('./run.mjs');
144
+ for (const { kind, count, first } of failing) {
145
+ if (!ledger.newWith(first)) continue;
146
+ const prs = first.changes?.prs ?? [];
147
+ const repo = first.changes?.repository;
148
+ if (!prs.length || !repo) continue;
149
+ const where = `changes:${first.firstRelease}`;
150
+ if (first.said.includes(where)) continue;
151
+ if (first.triedAt && Date.now() - Date.parse(first.triedAt) < 3_600_000) continue;
152
+ const body = [
153
+ `**supercode on ${machine}: a failure keeps happening** — \`${kind}\`, ${count} times in the last hour.`,
154
+ `It is new with supercode ${first.firstRelease} on this machine (first ${first.firstAt}, absent under the release before it), the release this pull request is in with ${prs.map((pr) => `#${pr.number}`).join(', ')}; it runs ${release?.version ?? 'an unknown release'} now.`,
155
+ 'Said by the machine-health pack from the machine\'s own records (sdk/health/lib/failures.mjs); its maintainer has the alarm too.',
156
+ ].join('\n\n');
157
+ let all = true;
158
+ for (const pr of prs) {
159
+ const posted = await run(gh, ['pr', 'comment', String(pr.number), '--repo', repo, '--body', body], { timeoutMs: 20_000, env });
160
+ if (!posted.ok) all = false;
161
+ }
162
+ if (all) ledger.said(kind, where);
163
+ else { first.triedAt = new Date().toISOString(); ledger.save(); }
164
+ }
165
+ }
package/lib/mail.mjs CHANGED
@@ -1,20 +1,74 @@
1
- // Alarm mail, filed in each subscriber's mailbox through supercode's filing door (`supercode message file`), from
2
- // `sc:<machine>:operator:machine-health`, and delivered by the mailbox (docs/adr/0008-agent-mailbox.md, decision 17;
3
- // D138 step 6). Filing is a local write: no door, no network, and nothing for the probe to retry. The mailbox resolves
4
- // an agent address (`sc:<machine>:agent:<name>`) to the agent's main session when it delivers, retries within its
5
- // bounds, and records what it could not deliver (`supercode message waiting`, `supercode message expired`).
6
- import { spawnSync } from 'node:child_process';
1
+ // Alarm mail, filed in each subscriber's mailbox by this machine's daemon (`harness.v1.mail.file`, its one filing,
2
+ // sdk/teams/mail/send.mjs), from `sc:<machine>:operator:machine-health`, and delivered by the mailbox
3
+ // (docs/adr/0008-agent-mailbox.md, decision 17; D138 step 6). Filing is a local write: one call on the daemon's local
4
+ // socket, no network, and nothing for the probe to retry. The mailbox resolves an agent address
5
+ // (`sc:<machine>:agent:<name>`) to the agent's main session when it delivers, retries within its bounds, and records what
6
+ // it could not deliver (`supercode message waiting`, `supercode message expired`). This package has no dependencies, so
7
+ // the call is made here with the daemon's own framing (sdk/volter-teams/channel.mjs).
8
+ import { createHash } from 'node:crypto';
9
+ import { connect } from 'node:net';
10
+ import { homedir } from 'node:os';
11
+ import { join } from 'node:path';
12
+
13
+ /** Where this OS user's daemon serves its local door (sdk/volter-teams/home.mjs `machineSocketPath`). */
14
+ export function daemonSocket(env = process.env) {
15
+ const home = env.SUPERCODE_HOME || (env.XDG_CONFIG_HOME ? join(env.XDG_CONFIG_HOME, 'supercode') : join(homedir(), '.config', 'supercode'));
16
+ const inside = join(env.SUPERCODE_TEAMS_HOME ?? join(home, 'teams'), 'machine.sock');
17
+ const hash = createHash('sha256').update(inside).digest('hex').slice(0, 24);
18
+ if (process.platform === 'win32') return `\\\\.\\pipe\\supercode-${hash}`;
19
+ if (Buffer.byteLength(inside) <= 100) return inside;
20
+ return join('/tmp', `supercode-machine-${typeof process.getuid === 'function' ? process.getuid() : 'user'}`, `${hash}.sock`);
21
+ }
22
+
23
+ /** One `harness.v1` call to this machine's daemon: a hello each way, the `node` stream, one request, its answer. */
24
+ export function daemonCall(method, params, { env = process.env, timeoutMs = 15_000 } = {}) {
25
+ return new Promise((resolve, reject) => {
26
+ const socket = connect(daemonSocket(env));
27
+ let buffer = Buffer.alloc(0), phase = 'hello', text = '';
28
+ const done = (error, value) => { clearTimeout(timer); socket.destroy(); error ? reject(error) : resolve(value); };
29
+ const timer = setTimeout(() => done(new Error(`this machine's daemon did not answer ${method} within ${timeoutMs / 1000} s`)), timeoutMs);
30
+ const send = (value) => { const bytes = Buffer.from(JSON.stringify(value)); const head = Buffer.alloc(4); head.writeUInt32BE(bytes.length, 0); socket.write(Buffer.concat([head, bytes])); };
31
+ socket.on('error', (error) => done(new Error(`no machine daemon is answering (${error.message}); run \`supercode teams machine start\``)));
32
+ socket.on('close', () => done(new Error('the daemon\'s socket closed before it answered')));
33
+ socket.on('connect', () => send({ p: 'supercode-machine/1', nv: 3 }));
34
+ socket.on('data', (chunk) => {
35
+ buffer = Buffer.concat([buffer, chunk]);
36
+ while (buffer.length >= 4 && buffer.length >= 4 + buffer.readUInt32BE(0)) {
37
+ const frame = JSON.parse(buffer.subarray(4, 4 + buffer.readUInt32BE(0)).toString('utf8'));
38
+ buffer = buffer.subarray(4 + buffer.readUInt32BE(0));
39
+ if (phase === 'hello') {
40
+ if (frame.error) return done(new Error(frame.error));
41
+ phase = 'open';
42
+ send({ ch: 1, t: 'open', name: 'node', params: {} });
43
+ continue;
44
+ }
45
+ if (frame.t === 'ping') { send({ ch: 0, t: 'pong' }); continue; }
46
+ if (frame.ch !== 1) continue;
47
+ if (frame.t === 'close') return done(new Error(`the daemon refused: ${frame.reason ?? 'closed'}`));
48
+ if (frame.t === 'opened') { send({ ch: 1, t: 'data', d: `${JSON.stringify({ id: 1, method, params })}\n` }); continue; }
49
+ if (frame.t !== 'data') continue;
50
+ text += frame.d;
51
+ for (let end; (end = text.indexOf('\n')) >= 0;) {
52
+ const line = text.slice(0, end); text = text.slice(end + 1);
53
+ let message = null; try { message = JSON.parse(line); } catch { continue; }
54
+ if (message?.id !== 1) continue;
55
+ return message.error ? done(new Error(message.error.message ?? 'the daemon refused')) : done(null, message.result);
56
+ }
57
+ }
58
+ });
59
+ });
60
+ }
7
61
 
8
62
  /** File one alarm mail for each address. Answers what was filed and the first failure, if any. */
9
- export function fileAlarm({ machine, env = process.env, to, key, subject, body }) {
63
+ export async function fileAlarm({ machine, env = process.env, to, key, subject, body }) {
10
64
  const filed = [];
11
65
  let error = null;
12
66
  for (const address of to) {
13
- const request = { from: `sc:${machine}:operator:machine-health`, from_name: 'Machine health', to: address, id: `${key}-${address}`, subject, body, wake: true };
14
- const done = spawnSync(env.SUPERCODE_BIN || 'supercode', ['message', 'file'], { env, input: JSON.stringify(request), encoding: 'utf8', timeout: 10_000, windowsHide: true, shell: process.platform === 'win32' });
15
- let answer = null; try { answer = JSON.parse(String(done.stdout ?? '').trim().split('\n').at(-1)); } catch { /* none */ }
67
+ let answer = null;
68
+ try { answer = await daemonCall('harness.v1.mail.file', { from: `sc:${machine}:operator:machine-health`, from_name: 'Machine health', to: address, id: `${key}-${address}`, subject, body, wake: true }, { env }); }
69
+ catch (failure) { error ??= `mail to ${address} was not filed: ${failure.message}`; continue; }
16
70
  if (answer?.code === 0) filed.push({ to: address, id: answer.message_id });
17
- else error ??= `mail to ${address} was not filed: ${answer?.text || String(done.stderr ?? '').trim().slice(-300) || done.error?.message || `exit ${done.status}`}`;
71
+ else error ??= `mail to ${address} was not filed: ${answer?.text ?? 'the daemon answered nothing readable'}`;
18
72
  }
19
73
  return { filed, error };
20
74
  }
package/lib/probe.mjs CHANGED
@@ -184,7 +184,7 @@ export class Probe {
184
184
  // The reading is written first; the mail is filed after it, and what the filing did is written beside it.
185
185
  writeAtomic(this.f.latest, snapshot);
186
186
  if (mail && to.length) {
187
- const result = fileAlarm({ machine: this.machine, env: this.env, to, key: `health-${this.machine}-${now}`, subject: mail.subject, body: mail.body });
187
+ const result = await fileAlarm({ machine: this.machine, env: this.env, to, key: `health-${this.machine}-${now}`, subject: mail.subject, body: mail.body });
188
188
  this.lastMail = { at: new Date(now).toISOString(), filed: result.filed.length, to: result.filed.map((f) => f.to), error: result.error };
189
189
  if (result.error) this.log(result.error);
190
190
  snapshot.mail = this.lastMail;
package/lib/sample.mjs CHANGED
@@ -4,6 +4,9 @@ import { cpus, homedir, loadavg } from 'node:os';
4
4
  import { existsSync, readFileSync, statSync } from 'node:fs';
5
5
  import { run } from './run.mjs';
6
6
  import { rendererOrigins, originOf, localPort } from './chrome.mjs';
7
+ import { expiredFailures, failureSummary, keyFor, Ledger, runningRelease, sayOnChanges, slowFailures } from './failures.mjs';
8
+ import { healthDir } from './paths.mjs';
9
+ import { join } from 'node:path';
7
10
 
8
11
  const MB = 1024 * 1024;
9
12
  const GB = 1024 * MB;
@@ -415,11 +418,12 @@ export class Sampler {
415
418
  // enrollment record, whose connection field is not kept current.
416
419
  // Liveness is the daemon's cheapest door (`describe`); `launches.list` walks every pane and can be slow
417
420
  // on a busy machine without the daemon being down, so its timeout is a reading, not an alarm.
418
- const [status, describe, launches, machines] = await Promise.all([
421
+ const [status, describe, launches, machines, expired] = await Promise.all([
419
422
  run(bin, ['teams', 'status', '--json'], { timeoutMs: 20_000, env: this.env }),
420
423
  run(bin, ['teams', 'describe'], { timeoutMs: 15_000, env: this.env }).then((r) => ({ ...r, ms: Date.now() - t0 })),
421
424
  run(bin, ['teams', 'launches', 'list'], { timeoutMs: 45_000, env: this.env }),
422
425
  run(bin, ['teams', 'machines', 'list', '--json'], { timeoutMs: 20_000, env: this.env }),
426
+ run(bin, ['message', 'expired', '--json'], { timeoutMs: 20_000, env: this.env }),
423
427
  ]);
424
428
  let described = null; try { described = JSON.parse(describe.stdout); } catch { /* the daemon did not answer */ }
425
429
  let data = null; try { data = JSON.parse(status.stdout).data; } catch { /* unreadable */ }
@@ -454,6 +458,33 @@ export class Sampler {
454
458
  if (Array.isArray(rows)) slow.observations.push({ covered: 'supercode.launch:', family: 'supercode', all: true }, { covered: 'supercode.launchFailed:', family: 'supercode', all: true });
455
459
  for (const r of stuck) slow.observations.push({ key: `supercode.launch:${r.key}`, family: 'supercode', def: { forSec: 0 }, value: 1, met: { warn: true, crit: false, clears: false }, summary: r.status === 'issued' ? `launch ${r.key} (${r.kind ?? 'pane'}) issued ${Math.round((now - r.at) / 60_000)} min ago and never became ready` : `launch ${r.key} is live but its first input was not delivered: ${r.inputError}` });
456
460
  for (const r of failed) slow.observations.push({ key: `supercode.launchFailed:${r.key}`, family: 'supercode', def: { forSec: 0 }, value: 1, met: { warn: true, crit: false, clears: false }, summary: `launch ${r.key} (${r.kind ?? 'pane'}) failed at ${new Date(r.at).toISOString()}` });
461
+
462
+ // the machine's own records of failure (t_b3844842): each kind that keeps happening is an alarm naming the release
463
+ // that first showed it here and that release's changes; a record that could not be read leaves its kinds unknown
464
+ const fdef = alarm('supercode.failing');
465
+ const since = now - fdef.windowMin * 60_000;
466
+ const kinds = new Map();
467
+ const unread = [];
468
+ try { for (const [kind, seen] of slowFailures(this.env, since)) kinds.set(kind, seen); } catch (error) { unread.push(`the slow log could not be read: ${error.message}`); }
469
+ let expiries = null; try { expiries = JSON.parse(expired.stdout); } catch { unread.push(`the mailbox's expiries could not be listed: ${expired.error || 'no answer'}`); }
470
+ if (Array.isArray(expiries)) for (const [kind, seen] of expiredFailures(expiries, since)) kinds.set(kind, seen);
471
+ if (unread.length) slow.observations.push({ unreadPrefix: 'supercode.failing:', family: 'supercode', why: unread.join('; ') });
472
+ else slow.observations.push({ covered: 'supercode.failing:', family: 'supercode', all: true });
473
+ // read again each cycle: an upgrade replaces this pack's files under a probe that keeps running
474
+ const release = runningRelease();
475
+ this.ledger ??= new Ledger(join(healthDir(this.env), 'failures.json'));
476
+ this.ledger.running(release, now);
477
+ const failing = [];
478
+ for (const [kind, seen] of kinds) {
479
+ const first = this.ledger.seen(kind, seen.last, release);
480
+ const isNew = this.ledger.newWith(first);
481
+ const summary = failureSummary({ kind, count: seen.count, windowMin: fdef.windowMin, source: seen.source }, first, release, isNew);
482
+ slow.observations.push({ key: keyFor(kind), family: 'supercode', def: fdef, value: seen.count, unit: `in ${fdef.windowMin} min`, summary, detail: { kind, firstRelease: first.firstRelease, seeded: first.seeded, newWithRelease: isNew, running: release?.version ?? null, prs: isNew ? first.changes?.prs ?? [] : [] } });
483
+ if (seen.count >= fdef.warn) failing.push({ kind, count: seen.count, first, summary });
484
+ }
485
+ this.ledger.settled();
486
+ slow.readings.supercode.failing = failing.map((f) => ({ kind: f.kind, count: f.count, firstRelease: f.first.firstRelease }));
487
+ await sayOnChanges({ failing, machine: this.machine, release, ledger: this.ledger, env: this.env, config: cfg });
457
488
  });
458
489
 
459
490
  if (on.has('browser-tabs')) await attempt('browser-tabs', async () => {
@@ -34,6 +34,9 @@ export const DEFAULTS = {
34
34
  'service.down': { forSec: 60 },
35
35
  'supercode.connector': { forSec: 60 },
36
36
  'supercode.launch': { stuckMin: 5, failedWithinMin: 30 },
37
+ // one failure that keeps happening (the same command or door with the same outcome, the same mail expiry) in the
38
+ // window; it clears when the window holds none of it (t_b3844842)
39
+ 'supercode.failing': { warn: 3, crit: 20, clear: 0, windowMin: 60, forSec: 0 },
37
40
  'browser.tabs': { warn: null },
38
41
  'power.thermal': { forSec: 60 },
39
42
  },
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@volter/supercode-health",
3
- "version": "0.1.6",
3
+ "version": "0.1.8",
4
4
  "type": "module",
5
5
  "description": "Optional machine pack: a cheap, deterministic probe of a machine's vitals that mails the machine's maintainer when a threshold is crossed. Each feature sits behind its own permission barrier.",
6
6
  "license": "MIT",