@volter/supercode-health 0.1.20 → 0.1.21

package/lib/promises.mjs

@@ -10,6 +10,11 @@
 // zero) and readable there the whole time. The probe (probe.mjs) runs the pass every slow interval and mails a false
 // promise to the `core` addresses on the pass that first reads it false; a release's activation watch runs the same
 // pass once the new daemon serves and compares it with the last pass before it.
+//
+// A pass taken for a release (`since`: when that release's daemon began to serve) counts only what happened under it.
+// A promise that reads an event (a board's last round, a mail's expiry) reads there only events that began at or after
+// `since`: a round that met no daemon while an install restarted it says nothing of the release that came up. Until
+// such an event exists the promise is unknown for that release, never true and never false.
 import { readFileSync, realpathSync, statfsSync, statSync, accessSync, constants } from 'node:fs';
 import { homedir, tmpdir, userInfo } from 'node:os';
 import { delimiter, isAbsolute, join } from 'node:path';
@@ -145,6 +150,28 @@ function dispatcherHomes(context, hours = 24) {
   });
 }
 
+/** A kept release's promises that have read true under it (activation time → names), as this process has read them. */
+const releaseHeldSeen = new Map();
+
+/** The promises that have read true under the release activated at `at`: from this process and from the probe's
+ *  snapshot, so it holds across a probe's restart and for a pass taken by hand. */
+function releaseHeld(context, at) {
+  const kept = json(join(healthDir(context.env), 'latest.json'))?.readings?.promises?.items?.find((item) => item.name === 'release-kept')?.detail;
+  return new Set(at == null ? [] : [...(kept?.at_ms === at ? kept.held ?? [] : []), ...(releaseHeldSeen.get(at) ?? [])]);
+}
+
+/** One promise of the list read inside its own bound: past it, or when its probe throws, it is unknown and says so. */
+async function readOne(entry, context) {
+  let timer;
+  const bound = new Promise((resolve) => { timer = setTimeout(() => resolve(unknown(`not read within its ${entry.bound_ms} ms bound`)), entry.bound_ms); });
+  const probe = PROBES[entry.probe];
+  const read = probe ? Promise.resolve().then(() => probe(context)).catch((error) => unknown(`could not be read: ${String(error.message ?? error).split('\n')[0].slice(0, 240)}`))
+    : Promise.resolve(unknown(`this release has no probe named ${entry.probe}`));
+  const result = await Promise.race([read, bound]);
+  clearTimeout(timer);
+  return result;
+}
+
 const PROBES = {
   async server(context) {
     const items = await machineList(context);
@@ -183,7 +210,7 @@ const PROBES = {
     const answer = await daemonCall('harness.v1.mail.expired', {}, { env: context.env, timeoutMs: 1300 });
     if (!Array.isArray(answer?.rows)) return no('the daemon answered its mail door with no list of expiries');
     if (record.mail_ready !== true) return no(`the daemon (pid ${record.pid}) answers and does not serve mail`);
-    const since = context.now - context.lines.mailExpiryMin * 60_000;
+    const since = Math.max(context.now - context.lines.mailExpiryMin * 60_000, context.since ?? 0);
     const recent = answer.rows.filter((row) => Number(row?.expired_at_ms) >= since);
     const reasons = [...new Set(recent.map((row) => String(row.reason ?? 'no reason given').slice(0, 80)))];
     const detail = { pid: record.pid, expired_recently: recent.length, reasons };
@@ -301,17 +328,20 @@ const PROBES = {
 
   async board(context) {
     const held = json(join(supercodeHome(context.env), 'board-homes.json')) ?? {};
-    const line = context.lines.boardRoundMin * 60_000, rounds = [], broken = [];
+    const line = context.lines.boardRoundMin * 60_000, rounds = [], broken = [], before = [];
     for (const root of new Set(Object.values(held).filter((value) => typeof value === 'string'))) {
       const beat = json(join(root, 'dispatcher-heartbeat.json'));
       if (!beat?.at) continue; // no dispatcher has run a round in this home
       const age = context.now - Date.parse(beat.at);
       const failed = (beat.boards ?? []).reduce((sum, board) => sum + (Number(board.failed) || 0), 0), errors = (beat.errors ?? []).length;
-      rounds.push({ home: root, round: beat.round ?? null, at: beat.at, failed, errors });
+      rounds.push({ home: root, round: beat.round ?? null, began: beat.began ?? null, at: beat.at, failed, errors });
+      // rounds that stopped are false whenever they are read; what a round met is read only of a round begun under `since`
       if (!(age <= line)) broken.push(`${root}: its last round (${beat.round ?? '?'}) ended ${ago(age)} ago`);
+      else if (context.since != null && !(Date.parse(beat.began ?? '') >= context.since)) before.push(`${root}: round ${beat.round ?? '?'} began before the daemon that serves now did`);
       else if (failed || errors) broken.push(`${root}: round ${beat.round ?? '?'} failed ${failed} and raised ${errors} error(s)`);
     }
     if (!rounds.length) return yes('no board dispatcher has run a round on this machine');
+    if (!broken.length && before.length) return unknown(`${before.join('; ')}; no round has begun and ended under it yet`);
     return broken.length ? no(broken.join('; '), rounds) : yes(rounds.map((r) => `${r.home}: round ${r.round ?? '?'} ${ago(context.now - Date.parse(r.at))} ago, failed 0`).join('; '), rounds);
   },
 
@@ -329,6 +359,11 @@ const PROBES = {
       : yes(rows.map((row) => `${row.folder}: ${row.freeGB} GB free`).join('; '), rows);
   },
 
+  // The watch's verdict (activation-watch.mjs) is one pass taken seconds after the daemon served, and is not the last
+  // word on a kept release. A promise it read false there, or could not read yet (a board whose first round under
+  // the release had not ended), is read again here as the watch reads it: under the release's daemon only. False is
+  // the charge; true ends it for this activation, kept from pass to pass (this process's memory and the last pass in
+  // the probe's snapshot) so nothing that fails later under the same release is laid at its activation.
   async activation(context) {
     const verdict = json(join(teamsHome(context.env), 'service', 'last-activation.json'));
     if (!verdict?.state) return yes('no activation is recorded on this machine');
@@ -338,30 +373,47 @@ const PROBES = {
       return no(`${release} was ${verdict.state === 'rolled_back' ? `put back at ${at}; ${verdict.restored?.release ?? 'the release before'} serves` : verdict.state === 'restarted' ? `stopped and started again at ${at}` : `stopped at ${at} and nothing was restored`}: ${verdict.reason ?? verdict.detail ?? 'no reason recorded'}`,
         { state: verdict.state, at_ms: verdict.at_ms ?? null, broken: verdict.promises?.broken ?? [] });
     }
-    const broken = verdict.promises?.broken ?? [];
-    if (broken.length) return no(`${release} was kept at ${at} and broke ${broken.map((b) => b.name).join(', ')}: ${verdict.promises.kept_because ?? 'it was not put back'}`, { state: verdict.state, at_ms: verdict.at_ms ?? null, broken });
-    return yes(`${release} ${verdict.state} at ${at}`);
+    const list = promiseList().promises;
+    const own = (row) => list.find((entry) => entry.name === row.name)?.probe === 'activation';
+    const charged = (verdict.promises?.broken ?? []).filter((row) => !own(row));
+    const awaited = (verdict.promises?.unread ?? []).filter((row) => row.before === 'true' && !own(row));
+    const held = releaseHeld(context, verdict.at_ms ?? null);
+    const under = { ...context, since: verdict.serving_since_ms ?? null };
+    const open = [...charged, ...awaited].filter((row) => !held.has(row.name));
+    const read = await Promise.all(open.map((row) => {
+      const entry = list.find((item) => item.name === row.name);
+      return entry ? readOne(entry, under) : unknown('this release has no promise of that name');
+    }));
+    const broken = [], unread = [];
+    open.forEach((row, i) => {
+      if (read[i].state === 'true') held.add(row.name);
+      else if (read[i].state === 'false') broken.push({ ...row, reading: read[i].reading });
+      else if (charged.includes(row)) broken.push(row); // unread now: what the watch read stands
+      else unread.push(row.name);
+    });
+    releaseHeldSeen.clear();
+    if (verdict.at_ms != null) releaseHeldSeen.set(verdict.at_ms, [...held]);
+    const detail = { state: verdict.state, at_ms: verdict.at_ms ?? null, broken, held: [...held], unread };
+    if (broken.length) return no(`${release} was kept at ${at} and broke ${broken.map((b) => b.name).join(', ')}: ${verdict.promises.kept_because ?? 'it was not put back'}`, detail);
+    const cleared = charged.filter((row) => held.has(row.name)).map((row) => row.name);
+    return yes(`${release} ${verdict.state} at ${at}${cleared.length ? `; ${cleared.join(', ')} read false at its activation and ${cleared.length === 1 ? 'holds' : 'hold'} under it` : ''}${unread.length ? `; ${unread.join(', ')} not read under it yet` : ''}`,
+      charged.length || awaited.length ? detail : undefined);
   },
 };
 
 /**
  * One pass: `{ version, at, machine, ms, items: [{ name, promise, state, reading, detail?, ms, bound_ms, activation }] }`.
  * Every probe runs at once; one that has not answered inside its bound is `unknown` and says so, and the pass ends
- * with the slowest probe. `only` reads some of the list by name.
+ * with the slowest probe. `only` reads some of the list by name. `since` (ms) takes the pass for the release whose
+ * daemon began to serve then: an event that began before it is not read.
  */
-export async function readPromises({ env = process.env, config = {}, machine = null, only = null, now = Date.now() } = {}) {
+export async function readPromises({ env = process.env, config = {}, machine = null, only = null, now = Date.now(), since = null } = {}) {
   const started = performance.now();
-  const context = { env, now, bin: env.SUPERCODE_BIN || 'supercode', machine: machineName(machine || env.SUPERCODE_HEALTH_MACHINE || undefined), lines: { ...PROMISE_DEFAULTS, ...(config.promises ?? {}) }, machines: null, processes: null };
+  const context = { env, now, since, bin: env.SUPERCODE_BIN || 'supercode', machine: machineName(machine || env.SUPERCODE_HEALTH_MACHINE || undefined), lines: { ...PROMISE_DEFAULTS, ...(config.promises ?? {}) }, machines: null, processes: null };
   const list = promiseList().promises.filter((entry) => !only || only.includes(entry.name));
   const items = await Promise.all(list.map(async (entry) => {
     const t0 = performance.now();
-    let timer;
-    const bound = new Promise((resolve) => { timer = setTimeout(() => resolve(unknown(`not read within its ${entry.bound_ms} ms bound`)), entry.bound_ms); });
-    const probe = PROBES[entry.probe];
-    const read = probe ? Promise.resolve().then(() => probe(context)).catch((error) => unknown(`could not be read: ${String(error.message ?? error).split('\n')[0].slice(0, 240)}`))
-      : Promise.resolve(unknown(`this release has no probe named ${entry.probe}`));
-    const result = await Promise.race([read, bound]);
-    clearTimeout(timer);
+    const result = await readOne(entry, context);
     return { name: entry.name, promise: entry.promise, ...result, ms: Math.round(performance.now() - t0), bound_ms: entry.bound_ms, activation: entry.activation };
   }));
   return { version: 1, at: new Date(now).toISOString(), machine: context.machine, ms: Math.round(performance.now() - started), items };

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@volter/supercode-health",
-  "version": "0.1.20",
+  "version": "0.1.21",
   "type": "module",
   "description": "Optional machine pack: a cheap, deterministic probe of a machine's vitals that mails the machine's maintainer when a threshold is crossed. Each feature sits behind its own permission barrier.",
   "license": "MIT",

package/promises.json

@@ -22,7 +22,7 @@
       "promise": "Where a board's dispatcher runs, its last round ended in the last ten minutes with nothing failed." },
     { "name": "disk-free", "probe": "disk", "bound_ms": 500, "activation": "report",
       "promise": "The volumes supercode's home, the person's home and the temporary folder are on each have more than 5 GB free." },
-    { "name": "release-kept", "probe": "activation", "bound_ms": 200, "activation": "report",
+    { "name": "release-kept", "probe": "activation", "bound_ms": 7000, "activation": "report",
       "promise": "The last release activated on this machine was kept: it was not put back, and it broke no promise that held before it." },
     { "name": "release-published", "probe": "release", "bound_ms": 5000, "activation": "report",
       "promise": "What main last tagged for release is published: no release has been built and waiting more than 30 minutes for the version npm installs to reach it. Read on the install's host, from the public registry." },