@bongos/core 1.19.954 → 1.19.956

package/tests/provisioning_runner_drift_notice.mjs

@@ -0,0 +1,292 @@
+// tests/provisioning_runner_drift_notice.mjs
+//
+// An intent-runner unit pointed at a different core than the instance installed now says
+// so, once, without anybody looking (task 1004000 — the follow-up the task 1002722 grade
+// asked for).
+//
+// THE DONE-WHEN IS TWO SENTENCES AND BOTH ARE HERE: "a unit repointed at a different core
+// than the installed one produces one notice within one tick, and a healthy host produces
+// none." The first is asserted through the real sweep tick, with a fake `systemctl`
+// reader, so what is proved is the wiring and not only the planner. The second is
+// asserted the same way — silence is a result, and the only way to know it is not the
+// silence of a check that never ran is to prove the tick DID run and chose to say
+// nothing.
+//
+// Also pinned, because each is a way this kind of alert goes wrong:
+//   • ONE notice, not one every five minutes — the state suppresses a repeat
+//   • a FIRST SIGHT is announced, unlike liveness's deliberate silence about an outage
+//     that predates the process (the reason is in runner-drift-notify.js's header, and a
+//     test that got this backwards would look just as green)
+//   • `unresolved` is NOT drift — it is the absence of a verdict, and announcing it would
+//     be a siren over nothing; it reaches the log line instead
+//   • no systemd → nothing said and no state touched, so a checker that cannot run never
+//     clears an announcement it also cannot re-make
+//   • a unit that VANISHES is not a recovery — "it is gone" and "it is fixed" are
+//     different facts and only one is good news
+//   • a drift check that throws does not take the liveness sweep with it
+//
+// DB-free and network-free: the systemctl reader and the notifier are injected.
+//
+// Run:  node tests/provisioning_runner_drift_notice.mjs
+
+import { strict as assert } from 'node:assert';
+import { createRequire } from 'node:module';
+import { readFileSync } from 'node:fs';
+
+const require = createRequire(import.meta.url);
+const notify = require('../modules/provisioning/pollers/runner-drift-notify.js');
+const sweep = require('../modules/provisioning/pollers/liveness-sweep.js');
+
+let passed = 0, failed = 0;
+function t(name, fn) {
+  try { fn(); passed++; console.log(`  PASS  ${name}`); }
+  catch (e) { failed++; console.log(`  FAIL  ${name}\n        ${e.message}`); }
+}
+async function ta(name, fn) {
+  try { await fn(); passed++; console.log(`  PASS  ${name}`); }
+  catch (e) { failed++; console.log(`  FAIL  ${name}\n        ${e.message}`); }
+}
+
+const DRIFTED = { name: 'bongos-intent-runner.service', status: 'drift', runnerVersion: '1.19.900', instanceVersion: '1.19.953', reason: 'runs core 1.19.900 but the instance installed 1.19.953' };
+const HEALTHY = { name: 'bongos-intent-runner.service', status: 'ok', runnerVersion: '1.19.953', instanceVersion: '1.19.953', reason: 'runs core 1.19.953' };
+const UNRESOLVED = { name: 'bongos-intent-runner.service', status: 'unresolved', runnerVersion: null, instanceVersion: '1.19.953', reason: 'ExecStart does not run a .js script directly' };
+
+console.log('the planner:');
+
+t('a drifted unit is announced on FIRST SIGHT, and the message names the installed version', () => {
+  const state = new Set();
+  const r = notify.planDriftNotices([DRIFTED], state);
+  assert.equal(r.notices.length, 1);
+  assert.equal(r.notices[0].kind, 'drift');
+  assert.match(r.notices[0].text, /older version/);
+  assert.match(r.notices[0].text, /1\.19\.953/, 'the installed version is in the message — it is what the runner should be on');
+  assert.match(r.notices[0].text, /bongos-intent-runner\.service/, 'and the unit name, once, for whoever fixes it');
+  assert.equal(r.drifting, 1);
+  // The opposite of liveness's rule, deliberately: drift that predates this process is
+  // exactly what nobody noticed, so a restart re-announcing it is correct.
+  assert.deepEqual([...state], ['bongos-intent-runner.service']);
+});
+
+t('the SECOND tick says nothing — one notice, not one every five minutes', () => {
+  const state = new Set();
+  notify.planDriftNotices([DRIFTED], state);
+  const again = notify.planDriftNotices([DRIFTED], state);
+  assert.deepEqual(again.notices, []);
+  assert.equal(again.drifting, 1, 'still counted in the account, just not re-announced');
+});
+
+t('a recovery is announced once, and then the unit is forgotten', () => {
+  const state = new Set();
+  notify.planDriftNotices([DRIFTED], state);
+  const fixed = notify.planDriftNotices([HEALTHY], state);
+  assert.equal(fixed.notices.length, 1);
+  assert.equal(fixed.notices[0].kind, 'clear');
+  assert.match(fixed.notices[0].text, /back on the current version/);
+  assert.deepEqual([...state], [], 'forgotten, so a later drift is a fresh first sight');
+  assert.deepEqual(notify.planDriftNotices([HEALTHY], state).notices, [], 'and the all-clear is not repeated either');
+});
+
+t('a unit that VANISHES is not a recovery', () => {
+  // "The unit is gone" and "the unit is fixed" are different facts and only one of them
+  // is good news. The state is still cleared, so a reinstated unit is a fresh first sight.
+  const state = new Set();
+  notify.planDriftNotices([DRIFTED], state);
+  const gone = notify.planDriftNotices([], state);
+  assert.deepEqual(gone.notices, [], 'no all-clear for a unit nobody fixed');
+  assert.deepEqual([...state], []);
+});
+
+t('several units drifting at once are ONE message, because they have one cause', () => {
+  const state = new Set();
+  const units = [DRIFTED, { ...DRIFTED, name: 'bongos-box-intent-runner.service' }];
+  const r = notify.planDriftNotices(units, state);
+  assert.equal(r.notices.length, 1, 'one message, not one per unit');
+  assert.match(r.notices[0].text, /2 background runners/);
+  assert.match(r.notices[0].text, /one upgrade that moved the code/, 'and it names the usual cause');
+  assert.equal(r.notices[0].units.length, 2);
+});
+
+t("'unresolved' is NOT drift — no notice, but it does reach the log line", () => {
+  const state = new Set();
+  const r = notify.planDriftNotices([UNRESOLVED], state);
+  assert.deepEqual(r.notices, [], 'the absence of a verdict is not a bad verdict');
+  assert.equal(r.unresolved, 1);
+  assert.equal(r.drifting, 0);
+  assert.deepEqual([...state], []);
+  // Visible without being urgent — and a host that is ALL unresolved must not read as
+  // healthy, which is why the line exists at all.
+  assert.match(notify.formatDriftLine(r), /unresolved=1 \(cannot tell — not a verdict\)/);
+});
+
+t('no systemd says nothing, touches no state, and reports systemd:false', () => {
+  const state = new Set(['bongos-intent-runner.service']);
+  for (const nothing of [null, undefined]) {
+    const r = notify.planDriftNotices(nothing, state);
+    assert.deepEqual(r.notices, []);
+    assert.equal(r.systemd, false);
+    assert.deepEqual([...state], ['bongos-intent-runner.service'],
+      'a checker that cannot run must not clear an announcement it also cannot re-make');
+  }
+});
+
+t('a healthy host produces no log line either — no narration every five minutes', () => {
+  const r = notify.planDriftNotices([HEALTHY], new Set());
+  assert.deepEqual(r.notices, []);
+  assert.equal(notify.formatDriftLine(r), null, 'a vanilla control plane stays quiet');
+  assert.equal(notify.formatDriftLine({ systemd: true, checked: 0, drifting: 0, unresolved: 0 }), null, 'and so does a host with no runner units');
+  assert.match(notify.formatDriftLine({ systemd: true, checked: 2, drifting: 1, unresolved: 0 }), /runners=2 · drift=1/);
+});
+
+console.log('\nthrough the real sweep tick:');
+
+// The sweep's own dependency seam, with no projects to probe: this tick is about the
+// runners. `collectRunnerDrift` is what the production path reads systemd through.
+function tickDeps({ units, notifySpy, logs }) {
+  return {
+    // `pool`, not `db` — sweepOnce's own seam name. Passing the wrong one silently falls
+    // back to the real api.pool and the tick tries to reach Postgres, which is how these
+    // five cases first failed.
+    pool: { query: async () => ({ rows: [] }) },
+    log: (m) => logs.push(String(m)),
+    notify: (n) => notifySpy.push(n),
+    collectRunnerDrift: async () => units,
+    instanceRoot: '/srv/instance',
+  };
+}
+
+// No active instances to probe: this tick is about the runners, and the roster read is
+// the only other thing sweepOnce does before reaching them.
+const provisioning = require('../modules/provisioning/provisioning.js');
+provisioning.listActiveInstancesForLiveness = async () => [];
+
+await ta('DONE-WHEN 1: a repointed unit produces ONE notice within ONE tick', async () => {
+  sweep.resetNoticeState();
+  const notifySpy = [], logs = [];
+  const out = await sweep.sweepOnce(tickDeps({ units: [DRIFTED], notifySpy, logs }));
+  assert.equal(notifySpy.length, 1, `one notice from one tick, got ${notifySpy.length}`);
+  assert.equal(notifySpy[0].kind, 'drift');
+  assert.equal(out.runnerDrift, 1, 'and the tick reports it in its account');
+  assert.ok(logs.some((l) => /\[drift\].*drift=1/.test(l)), 'the journal carries the line too');
+});
+
+await ta('DONE-WHEN 2: a healthy host produces NONE — and the tick demonstrably ran', async () => {
+  sweep.resetNoticeState();
+  const notifySpy = [], logs = [];
+  let asked = 0;
+  const deps = tickDeps({ units: [HEALTHY], notifySpy, logs });
+  const out = await sweep.sweepOnce({ ...deps, collectRunnerDrift: async () => { asked++; return [HEALTHY]; } });
+  assert.equal(notifySpy.length, 0, 'nothing is said');
+  assert.equal(asked, 1, 'and this is NOT the silence of a check that never ran');
+  assert.equal(out.runnerDrift, 0);
+  assert.ok(!logs.some((l) => /\[drift\]/.test(l)), 'no line either — silence all the way down');
+});
+
+await ta('the same drift over two ticks is announced once', async () => {
+  sweep.resetNoticeState();
+  const notifySpy = [], logs = [];
+  const deps = tickDeps({ units: [DRIFTED], notifySpy, logs });
+  await sweep.sweepOnce(deps);
+  await sweep.sweepOnce(deps);
+  assert.equal(notifySpy.length, 1, `the second tick must not re-announce, got ${notifySpy.length}`);
+});
+
+await ta('a drift check that THROWS does not take the sweep with it', async () => {
+  sweep.resetNoticeState();
+  const notifySpy = [], logs = [];
+  const deps = tickDeps({ units: [], notifySpy, logs });
+  const out = await sweep.sweepOnce({ ...deps, collectRunnerDrift: async () => { throw new Error('systemctl exploded'); } });
+  assert.ok(out, 'the tick still returns its liveness account');
+  assert.equal(out.swept, 0);
+  // Logged, not swallowed: silence here is indistinguishable from a healthy host, which
+  // is the exact failure this task was filed about.
+  assert.ok(logs.some((l) => /\[drift\] check failed/.test(l) && /systemctl exploded/.test(l)),
+    `the failure is said out loud: ${logs.join(' | ')}`);
+});
+
+await ta('the check is not gated on there being tenant projects', async () => {
+  // A control plane with no tenants still has runners worth checking — and that is the
+  // host this runs on.
+  sweep.resetNoticeState();
+  const notifySpy = [], logs = [];
+  const out = await sweep.sweepOnce(tickDeps({ units: [DRIFTED], notifySpy, logs }));
+  assert.equal(out.swept, 0, 'no projects at all');
+  assert.equal(notifySpy.length, 1, 'and the drift was still found');
+});
+
+console.log('\nwiring:');
+
+t('the module crosses NO require boundary into core — the checker is a subprocess', () => {
+  // A module may require only src/module-api.js (ADR 0083, docs/modules-contract.md, and
+  // this module's own CLAUDE.md header). scripts/ is core-package territory, so the
+  // checker is read the way the sibling reads scripts/discord/post.js: as a subprocess.
+  // fitness's checkModulesImportOnlyApi does not currently catch a scripts/ target, so
+  // this assertion is the only thing standing between the rule and a silent regression.
+  for (const f of ['runner-drift-notify.js', 'liveness-sweep.js']) {
+    const src = readFileSync(new URL(`../modules/provisioning/pollers/${f}`, import.meta.url), 'utf8');
+    const requires = [...src.matchAll(/require\((['"])([^'"]+)\1\)/g)].map((m) => m[2]);
+    for (const spec of requires) {
+      // src/module-api(.js) is THE doorway and is allowed; everything else under src/ or
+      // scripts/ is not.
+      if (/^(\.\.\/)+src\/module-api(\.js)?$/.test(spec)) continue;
+      assert.ok(!/(^|\/)scripts\//.test(spec) && !/(^|\/)src\//.test(spec),
+        `${f} requires "${spec}" — a module may require only src/module-api.js; spawn a script instead (ADR 0083)`);
+    }
+  }
+  const notifySrc = readFileSync(new URL('../modules/provisioning/pollers/runner-drift-notify.js', import.meta.url), 'utf8');
+  assert.match(notifySrc, /execFile/, 'the checker is spawned');
+  assert.match(notifySrc, /runner-drift\.js', '--instance-root'|scripts\/gds\/runner-drift\.js/, 'and it is the checker that is spawned');
+});
+
+t('postNotice is NOT re-exported — the sweep takes it straight from its owner', () => {
+  // A re-export with no production caller is API surface to keep true for nothing.
+  assert.equal(notify.postNotice, undefined);
+  const sweepSrc = readFileSync(new URL('../modules/provisioning/pollers/liveness-sweep.js', import.meta.url), 'utf8');
+  assert.match(sweepSrc, /const \{ planNotices, postNotice \} = require\('\.\/liveness-notify'\);/,
+    'one Discord path, imported from the file that owns it');
+});
+
+await ta('readRunnerDrift: a NON-ZERO exit is the drift signal, not a failed read', async () => {
+  // runner-drift.js exits 1 exactly WHEN it finds drift — that is its contract as a
+  // CI-style gate. Treating a failed exit as a failed read would discard the only result
+  // worth reporting, so the exit code is ignored whenever stdout parses.
+  const payload = JSON.stringify({ instanceRoot: '/srv/instance', systemd: true, units: [DRIFTED] });
+  const execFileImpl = (_cmd, _args, _opts, cb) => cb(Object.assign(new Error('Command failed'), { code: 1 }), payload, '');
+  const units = await notify.readRunnerDrift({ instanceRoot: '/srv/instance', execFileImpl });
+  assert.equal(units.length, 1);
+  assert.equal(units[0].status, 'drift');
+});
+
+await ta('readRunnerDrift: it spawns the checker with the instance root, from the repo root', async () => {
+  let seen = null;
+  const execFileImpl = (cmd, args, opts, cb) => { seen = { cmd, args, opts }; cb(null, JSON.stringify({ systemd: true, units: [] }), ''); };
+  const units = await notify.readRunnerDrift({ instanceRoot: '/srv/instance', execFileImpl });
+  assert.deepEqual(units, []);
+  assert.equal(seen.cmd, 'node');
+  assert.deepEqual(seen.args, ['scripts/gds/runner-drift.js', '--instance-root', '/srv/instance', '--json']);
+  assert.match(seen.opts.cwd, /cloud-bongos/, 'invoked from the repo root regardless of cwd');
+  assert.ok(seen.opts.timeout > 0, 'a hung systemctl costs the check, not the tick');
+});
+
+await ta('readRunnerDrift: "could not look" is NULL, never an empty array', async () => {
+  // "No runner units" and "could not ask" must not read the same — planDriftNotices
+  // treats them differently, and an empty array would silently clear announcements.
+  const cases = [
+    ['unparseable stdout', (_c, _a, _o, cb) => cb(null, 'systemctl: command not found', '')],
+    ['no systemd', (_c, _a, _o, cb) => cb(null, JSON.stringify({ systemd: false, units: [] }), '')],
+    ['empty stdout', (_c, _a, _o, cb) => cb(new Error('timeout'), '', '')],
+    ['a throwing spawn', () => { throw new Error('ENOENT'); }],
+  ];
+  for (const [why, execFileImpl] of cases) {
+    assert.equal(await notify.readRunnerDrift({ instanceRoot: '/x', execFileImpl }), null, why);
+  }
+  // And the empty-array case really is distinct: a host with systemd and no units.
+  assert.deepEqual(await notify.readRunnerDrift({ instanceRoot: '/x', execFileImpl: (_c, _a, _o, cb) => cb(null, JSON.stringify({ systemd: true, units: [] }), '') }), []);
+});
+
+t('the sweep clears BOTH notice states on reset, or a test leaks into the next one', () => {
+  const src = readFileSync(new URL('../modules/provisioning/pollers/liveness-sweep.js', import.meta.url), 'utf8');
+  assert.match(src, /function resetNoticeState\(\) \{ noticeState\.clear\(\); driftState\.clear\(\); \}/);
+});
+
+console.log(`\nprovisioning_runner_drift_notice: ${passed} passed, ${failed} failed`);
+if (failed) process.exit(1);