@bongos/core 1.19.82 → 1.19.84

package/modules/provisioning/tests/liveness.mjs

@@ -0,0 +1,420 @@
+// modules/provisioning/tests/liveness.mjs
+//
+// The VERIFICATION OVERLAY (task 1002696; docs/design/projects-surfaces-redesign-
+// direction.md §3): the heartbeat sweep that probes every active instance's
+// /healthz and records what it SAW, and the API exposure of the two columns it
+// writes. Covers:
+//   • healthzUrl / formatSweepLine — the pure probe-target + disposition-line rules
+//   • publicInstance — never-probed (null) stays distinct from probed-and-down
+//   • sweepOnce — an answering instance is up, a refusing/timing-out one is down,
+//     both columns are written for both, and `status` is NEVER touched
+//   • GET /provisioning/instances + /:id — the two fields ride the API
+//   • POST /provisioning/instances — an 'error' instance re-enqueues (Try again)
+//
+// DB-free: `db` is a recording fake and the data-access fns the ROUTES use are
+// stubbed on the cached module object (the tests/provisioning_domain_attach.mjs
+// pattern). The only network is loopback — ephemeral servers this file starts and
+// stops itself, so a probe result is real rather than mocked.
+//
+// Run standalone: node modules/provisioning/tests/liveness.mjs
+
+import { strict as assert } from 'node:assert';
+import { createRequire } from 'node:module';
+import http from 'node:http';
+
+const require = createRequire(import.meta.url);
+const provisioning = require('../provisioning.js');
+const liveness = require('../pollers/liveness-sweep.js');
+
+let passed = 0, failed = 0;
+function t(name, fn) {
+  try { fn(); passed++; console.log(`  PASS  ${name}`); }
+  catch (e) { failed++; console.log(`  FAIL  ${name}\n        ${e.message}`); }
+}
+async function ta(name, fn) {
+  try { await fn(); passed++; console.log(`  PASS  ${name}`); }
+  catch (e) { failed++; console.log(`  FAIL  ${name}\n        ${e.message}`); }
+}
+
+function listen(server) {
+  return new Promise((r) => server.listen(0, '127.0.0.1', r));
+}
+// Destroy keep-alive sockets before closing — fetch's connection pool would
+// otherwise hold them open past the last assertion and stall the exit.
+function closeServer(server) {
+  if (server.closeAllConnections) server.closeAllConnections();
+  server.close();
+}
+// A port nothing is listening on — bind one, then give it back.
+async function closedPort() {
+  const s = http.createServer();
+  await listen(s);
+  const p = s.address().port;
+  await new Promise((r) => s.close(r));
+  return p;
+}
+// A fake db that answers the sweep's two statements and records every write.
+function fakeDb(rows) {
+  const updates = [];
+  return {
+    updates,
+    query: async (sql, params) => {
+      if (/SELECT id, slug, domain, port FROM provisioning_instances/.test(sql)) return { rows };
+      if (/UPDATE provisioning_instances/.test(sql)) {
+        updates.push({ sql, id: params[0], up: params[1] });
+        return { rows: [{ id: params[0], last_verified_at: new Date(), last_verified_up: params[1] }] };
+      }
+      throw new Error(`unexpected SQL: ${sql}`);
+    },
+  };
+}
+
+console.log('healthzUrl (where the heartbeat knocks):');
+
+t('prefers the public address (https), falls back to the recorded loopback port', () => {
+  assert.equal(liveness.healthzUrl({ domain: 'Mercury.Example.com.', port: 3101 }), 'https://mercury.example.com/healthz');
+  assert.equal(liveness.healthzUrl({ domain: null, port: 3101 }), 'http://127.0.0.1:3101/healthz');
+});
+
+t('an unaddressed, portless (or junk-addressed) instance is unprobeable — null, never a guessed URL', () => {
+  assert.equal(liveness.healthzUrl({ domain: null, port: null }), null);
+  assert.equal(liveness.healthzUrl({ domain: 'not a host', port: 0 }), null);
+  assert.equal(liveness.healthzUrl({ domain: 'com', port: 99999 }), null); // bare TLD + out-of-range port
+});
+
+console.log('\nformatSweepLine (the one-line per-sweep account):');
+
+t('renders the disposition line, naming each down instance and its reason', () => {
+  assert.equal(
+    liveness.formatSweepLine({ swept: 6, up: 4, down: 2, reasons: ['mercury:ECONNREFUSED', 'swag:timeout'] }),
+    '[liveness] swept 6 active: 4 up · 2 down (mercury:ECONNREFUSED, swag:timeout)'
+  );
+});
+
+t('an all-green sweep carries no parenthetical; unaddressed instances are counted separately', () => {
+  assert.equal(liveness.formatSweepLine({ swept: 3, up: 3, down: 0 }), '[liveness] swept 3 active: 3 up · 0 down');
+  assert.equal(
+    liveness.formatSweepLine({ swept: 4, up: 3, down: 0, unprobeable: 1 }),
+    '[liveness] swept 4 active: 3 up · 0 down · 1 unaddressed'
+  );
+});
+
+console.log('\npublicInstance (the overlay projection):');
+
+t('never-probed (null) stays distinct from probed-and-down (false)', () => {
+  const never = provisioning.publicInstance({ id: 1, slug: 'a', status: 'active' });
+  assert.equal(never.last_verified_at, null);
+  assert.equal(never.last_verified_up, null, 'unverified is stated as unverified, never as down');
+  const down = provisioning.publicInstance({ id: 2, slug: 'b', status: 'active', last_verified_at: '2026-08-12T10:00:00Z', last_verified_up: false });
+  assert.equal(down.last_verified_at, '2026-08-12T10:00:00Z');
+  assert.equal(down.last_verified_up, false);
+});
+
+console.log('\nsweepOnce (real loopback probes, recording fake db):');
+
+await ta('an answering instance is UP and a refusing one is DOWN — both columns written, status never touched', async () => {
+  const server = http.createServer((req, res) => {
+    if (req.url === '/healthz') { res.writeHead(200, { 'content-type': 'application/json' }); res.end('{"ok":true}'); return; }
+    res.writeHead(404); res.end();
+  });
+  await listen(server);
+  const rows = [
+    { id: 1, slug: 'mercury', domain: null, port: server.address().port },
+    { id: 2, slug: 'swag', domain: null, port: await closedPort() },
+  ];
+  const db = fakeDb(rows);
+  const lines = [];
+  try {
+    const out = await liveness.sweepOnce({ pool: db, log: (l) => lines.push(l), timeoutMs: 3000 });
+    assert.deepEqual(
+      { swept: out.swept, up: out.up, down: out.down, unprobeable: out.unprobeable },
+      { swept: 2, up: 1, down: 1, unprobeable: 0 }
+    );
+    assert.deepEqual(db.updates.map((u) => [u.id, u.up]).sort(), [[1, true], [2, false]]);
+    for (const u of db.updates) {
+      assert.match(u.sql, /SET last_verified_at = now\(\), last_verified_up = \$2/);
+      assert.ok(!/status/.test(u.sql), 'verification is an overlay — the sweep must never write status');
+    }
+    assert.match(lines[0], /^\[liveness\] swept 2 active: 1 up · 1 down \(swag:/);
+  } finally {
+    closeServer(server);
+  }
+});
+
+await ta('a hung instance times out (down), and a non-2xx answer is down too — not "live"', async () => {
+  const hung = http.createServer(() => { /* never responds */ });
+  const sick = http.createServer((_req, res) => { res.writeHead(503); res.end(); });
+  await listen(hung);
+  await listen(sick);
+  const rows = [
+    { id: 3, slug: 'hung', domain: null, port: hung.address().port },
+    { id: 4, slug: 'sick', domain: null, port: sick.address().port },
+  ];
+  const db = fakeDb(rows);
+  try {
+    const out = await liveness.sweepOnce({ pool: db, log: () => {}, timeoutMs: 300 });
+    assert.equal(out.up, 0);
+    assert.equal(out.down, 2);
+    assert.deepEqual(out.reasons.sort(), ['hung:timeout', 'sick:http 503']);
+    assert.deepEqual(db.updates.map((u) => u.up), [false, false]);
+  } finally {
+    closeServer(hung);
+    closeServer(sick);
+  }
+});
+
+await ta('an unaddressed active instance is left alone — no verdict is recorded for a probe never made', async () => {
+  const db = fakeDb([{ id: 5, slug: 'newborn', domain: null, port: null }]);
+  const lines = [];
+  const out = await liveness.sweepOnce({ pool: db, log: (l) => lines.push(l) });
+  assert.deepEqual({ swept: out.swept, up: out.up, down: out.down, unprobeable: out.unprobeable },
+    { swept: 1, up: 0, down: 0, unprobeable: 1 });
+  assert.equal(db.updates.length, 0, 'nothing written');
+  assert.equal(lines[0], '[liveness] swept 1 active: 0 up · 0 down · 1 unaddressed');
+});
+
+await ta('a control plane with no active instances sweeps silently (no 5-minute empty account)', async () => {
+  const db = fakeDb([]);
+  const lines = [];
+  const out = await liveness.sweepOnce({ pool: db, log: (l) => lines.push(l) });
+  assert.equal(out.swept, 0);
+  assert.equal(lines.length, 0);
+});
+
+console.log('\nSSRF gate (domain probes never reach private/reserved networks):');
+
+t('private/reserved addresses are recognized; public ones are not', () => {
+  for (const bad of ['127.0.0.1', '10.1.2.3', '172.16.0.9', '172.31.255.1', '192.168.1.1',
+    '169.254.169.254', '100.64.0.1', '198.18.0.1', '0.0.0.0', '::1', '::',
+    'fe80::1', 'fd00::5', '::ffff:10.0.0.5', 'garbage']) {
+    assert.equal(liveness.isPrivateAddress(bad), true, `${bad} must be private/refused`);
+  }
+  for (const ok of ['93.184.216.34', '161.35.132.201', '2606:2800:220:1:248:1893:25c8:1946']) {
+    assert.equal(liveness.isPrivateAddress(ok), false, `${ok} must be public`);
+  }
+});
+
+await ta('an IP-literal domain (cloud metadata) is blocked — no lookup, no probe, no verdict', async () => {
+  const db = fakeDb([{ id: 6, slug: 'evil', domain: '169.254.169.254', port: null }]);
+  const lines = [];
+  const out = await liveness.sweepOnce({
+    pool: db,
+    log: (l) => lines.push(l),
+    lookup: async () => { throw new Error('lookup must not run for an IP literal'); },
+    fetchImpl: async () => { throw new Error('the probe must never fire'); },
+  });
+  assert.deepEqual({ blocked: out.blocked, down: out.down, up: out.up }, { blocked: 1, down: 0, up: 0 });
+  assert.equal(db.updates.length, 0, 'a blocked probe records NO verdict (no oracle)');
+  assert.match(lines[0], /1 blocked \(evil:ip-literal\)/);
+});
+
+await ta('a domain RESOLVING to a private address is blocked the same way', async () => {
+  const db = fakeDb([{ id: 7, slug: 'sneaky', domain: 'internal.example.com', port: null }]);
+  const out = await liveness.sweepOnce({
+    pool: db,
+    log: () => {},
+    lookup: async () => [{ address: '10.0.0.5', family: 4 }],
+    fetchImpl: async () => { throw new Error('the probe must never fire'); },
+  });
+  assert.equal(out.blocked, 1);
+  assert.deepEqual(out.blockedReasons, ['sneaky:private-address']);
+  assert.equal(db.updates.length, 0);
+});
+
+await ta('an unresolvable domain still probes and records an honest DOWN (the qa.test case)', async () => {
+  const db = fakeDb([{ id: 8, slug: 'ghostly', domain: 'ghostly.qa.test', port: null }]);
+  const enotfound = Object.assign(new Error('getaddrinfo ENOTFOUND'), { code: 'ENOTFOUND' });
+  const out = await liveness.sweepOnce({
+    pool: db,
+    log: () => {},
+    lookup: async () => { throw enotfound; },
+    fetchImpl: async () => { throw Object.assign(new TypeError('fetch failed'), { cause: { code: 'ENOTFOUND' } }); },
+  });
+  assert.deepEqual({ blocked: out.blocked, down: out.down }, { blocked: 0, down: 1 });
+  assert.deepEqual(out.reasons, ['ghostly:ENOTFOUND']);
+  assert.deepEqual(db.updates.map((u) => u.up), [false], 'DNS-less domain = honest down, not a silent skip');
+});
+
+await ta('the gate hands back the address it vetted, so the probe can pin to it (no re-resolution)', async () => {
+  const gate = await liveness.domainProbeGate('mercury.example.com', {
+    lookup: async () => [{ address: '93.184.216.34', family: 4 }],
+  });
+  assert.equal(gate.allowed, true);
+  assert.deepEqual(gate.pin, { address: '93.184.216.34', family: 4 },
+    'the vetted address rides back — DNS must not be consulted twice (rebinding TOCTOU)');
+});
+
+await ta('a rebinding nameserver cannot re-aim the probe — the pinned address is what gets connected', async () => {
+  // The gate sees a public address; a second resolution would return metadata.
+  // The probe must connect to the FIRST one. We assert by capturing what the
+  // dispatcher's pinned lookup resolves to.
+  let answered = 0;
+  const rebinding = async () => {
+    answered += 1;
+    return [{ address: answered === 1 ? '93.184.216.34' : '169.254.169.254', family: 4 }];
+  };
+  const gate = await liveness.domainProbeGate('rebind.example.com', { lookup: rebinding });
+  assert.equal(gate.allowed, true);
+  assert.equal(gate.pin.address, '93.184.216.34');
+  const connected = [];
+  await liveness.probeHealthz('https://rebind.example.com/healthz', {
+    timeoutMs: 500,
+    fetchImpl: async (_url, opts) => {
+      // stand in for undici: resolve through the dispatcher's pinned lookup
+      assert.ok(!opts.dispatcher, 'a stubbed fetch skips the real dispatcher');
+      connected.push(gate.pin.address);
+      throw Object.assign(new TypeError('fetch failed'), { cause: { code: 'ECONNREFUSED' } });
+    },
+    pin: gate.pin,
+  });
+  assert.deepEqual(connected, ['93.184.216.34'], 'the probe uses the vetted address, never the second answer');
+  assert.equal(answered, 1, 'exactly one DNS resolution happens for the whole probe');
+});
+
+await ta('the pin is load-bearing on the REAL client: same URL, the pinned address decides where it lands', async () => {
+  const server = http.createServer((req, res) => {
+    if (req.url === '/healthz') { res.writeHead(200); res.end('{"ok":true}'); return; }
+    res.writeHead(404); res.end();
+  });
+  await listen(server);
+  const url = `http://pinned.invalid:${server.address().port}/healthz`;
+  try {
+    // 'pinned.invalid' resolves nowhere; only the pin can make this connect.
+    const hit = await liveness.probeHealthz(url, { timeoutMs: 3000, pin: { address: '127.0.0.1', family: 4 } });
+    assert.deepEqual(hit, { up: true }, 'the pinned address is what gets connected');
+    const miss = await liveness.probeHealthz(url, { timeoutMs: 3000, pin: { address: '127.0.0.2', family: 4 } });
+    assert.equal(miss.up, false, 'a different pin cannot reach the same server — the pin, not DNS, decides');
+  } finally {
+    closeServer(server);
+  }
+});
+
+await ta('a redirecting healthz is DOWN — the probe never follows a redirect (SSRF pivot)', async () => {
+  const bouncer = http.createServer((_req, res) => {
+    res.writeHead(302, { location: 'http://169.254.169.254/latest/meta-data/' });
+    res.end();
+  });
+  await listen(bouncer);
+  try {
+    const probe = await liveness.probeHealthz(`http://127.0.0.1:${bouncer.address().port}/healthz`, { timeoutMs: 2000 });
+    assert.equal(probe.up, false);
+    assert.equal(probe.reason, 'http 302');
+  } finally {
+    closeServer(bouncer);
+  }
+});
+
+console.log('\nshape-aware onboard plan (direction §7 — no promises a shape cannot keep):');
+
+t('runner-performed legs are absent from the self-host plan; control-plane shapes keep everything', () => {
+  const { onboardPlan, stepsForShape } = require('../onboard-plan.js');
+  const all = onboardPlan('greenfield');
+  const standalone = stepsForShape(all, 'standalone');
+  assert.equal(standalone.length, all.length, 'a control-plane shape gets the full sequence');
+  const selfHost = stepsForShape(all, 'self-host');
+  const selfKeys = selfHost.map((s) => s.key);
+  for (const runnerLeg of ['provision', 'oauth-place', 'dns-preflight']) {
+    assert.ok(!selfKeys.includes(runnerLeg), `${runnerLeg} assumes the platform runner — absent on self-host`);
+  }
+  for (const human of ['github-repo', 'oauth-app', 'first-sign-in', 'enter-hall']) {
+    assert.ok(selfKeys.includes(human), `${human} is the owner's own leg — present on every shape`);
+  }
+  assert.equal(stepsForShape(all, 'nonsense').length, all.length, 'unknown shape = unfiltered, never an empty plan');
+});
+
+// ── the routes, over a real mounted router (stubbed data access) ─────────────────
+let instance = null;
+let createResult = null;
+const enqueueCalls = [];
+provisioning.listInstancesForOwner = async () => (instance ? [instance] : []);
+provisioning.getInstanceById = async (_db, id) => (instance ? { ...instance, id } : null);
+provisioning.getOpenIntent = async () => null;
+provisioning.createInstance = async () => createResult;
+provisioning.enqueueIntent = async (_db, id, action) => {
+  enqueueCalls.push({ id, action });
+  return { intent: { id: 9, action, state: 'pending' }, created: true };
+};
+provisioning.recordEvent = async () => {};
+
+const api = require('../../../src/module-api.js');
+api.requireBuilder = (req, _res, next) => { req.builder = { id: 7, github_login: 'owner' }; next(); };
+api.requirePermission = () => (_req, _res, next) => next();
+
+const express = require('express');
+const provisioningRoutes = require('../routes/provisioning.js');
+const app = express();
+app.use(express.json());
+app.use((_req, res, next) => {
+  res.fail = (code, opts, details) => {
+    const status = typeof opts === 'number' ? opts : (opts && opts.status) || 400;
+    res.status(status).json({ error: { code, ...(typeof opts === 'object' ? opts : {}), ...(details || {}) } });
+  };
+  next();
+});
+app.use('/api/bongos', provisioningRoutes());
+
+const server = app.listen(0);
+await new Promise((r) => server.once('listening', r));
+const base = `http://127.0.0.1:${server.address().port}/api/bongos`;
+
+async function req(method, path, body) {
+  const res = await fetch(`${base}${path}`, {
+    method,
+    headers: body ? { 'content-type': 'application/json' } : {},
+    body: body ? JSON.stringify(body) : undefined,
+  });
+  const raw = await res.text();
+  let json = null;
+  try { json = JSON.parse(raw); } catch { /* non-JSON */ }
+  return { status: res.status, body: json, raw };
+}
+
+console.log('\nAPI exposure — the overlay rides the instance reads:');
+
+await ta('GET /provisioning/instances carries last_verified_at + last_verified_up', async () => {
+  instance = {
+    id: 18, owner_builder_id: 7, slug: 'mercury', status: 'active', hosting_shape: 'standalone',
+    domain: 'mercury.cloudbongos.com', port: 3101,
+    last_verified_at: '2026-08-12T10:00:00.000Z', last_verified_up: true,
+  };
+  const res = await req('GET', '/provisioning/instances');
+  assert.equal(res.status, 200, res.raw);
+  const [row] = res.body.instances;
+  assert.equal(row.last_verified_at, '2026-08-12T10:00:00.000Z');
+  assert.equal(row.last_verified_up, true);
+});
+
+await ta('GET /provisioning/instances/:id carries them too — including the never-probed nulls', async () => {
+  instance = { id: 19, owner_builder_id: 7, slug: 'fresh', status: 'active', domain: null, port: 3102 };
+  const res = await req('GET', '/provisioning/instances/19');
+  assert.equal(res.status, 200, res.raw);
+  assert.ok('last_verified_at' in res.body.instance, 'the field is always present, even unset');
+  assert.equal(res.body.instance.last_verified_at, null);
+  assert.equal(res.body.instance.last_verified_up, null);
+});
+
+console.log('\n"Try again" — re-enqueue a snagged instance (redesign direction §5.3):');
+
+t("PROVISION_ENQUEUE_FROM allows 'error' (the state a snagged card is in)", () => {
+  assert.ok(provisioning.PROVISION_ENQUEUE_FROM.has('error'));
+});
+
+await ta("POST /provisioning/instances on an 'error' instance re-enqueues a provision intent", async () => {
+  enqueueCalls.length = 0;
+  createResult = {
+    instance: { id: 20, owner_builder_id: 7, slug: 'snagged', status: 'error', hosting_shape: 'standalone', domain: null },
+    created: false,
+  };
+  const res = await req('POST', '/provisioning/instances', { slug: 'snagged' });
+  assert.equal(res.status, 200, res.raw);
+  assert.equal(res.body.queued, true, 'Try again actually re-enqueues');
+  assert.deepEqual(enqueueCalls, [{ id: 20, action: 'provision' }]);
+});
+
+server.close();
+
+console.log(`\nprovisioning liveness overlay: ${passed} passed, ${failed} failed`);
+// exitCode, not process.exit(): every server above is closed, so the loop drains
+// on its own — and exiting hard while a socket handle is mid-close aborts node on
+// Windows (libuv UV_HANDLE_CLOSING assertion).
+process.exitCode = failed ? 1 : 0;

package/package-lock.json

@@ -1,12 +1,12 @@
 {
   "name": "@bongos/core",
-  "version": "1.19.82",
+  "version": "1.19.84",
   "lockfileVersion": 3,
   "requires": true,
   "packages": {
     "": {
       "name": "@bongos/core",
-      "version": "1.19.82",
+      "version": "1.19.84",
       "license": "AGPL-3.0-or-later",
       "dependencies": {
         "express": "^4.21.2",

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@bongos/core",
-  "version": "1.19.82",
+  "version": "1.19.84",
   "description": "Cloud Bongos — the AI-first build platform core (GDS + platform surfaces + module system), installed as a versioned dependency (ADR 0108).",
   "license": "AGPL-3.0-or-later",
   "main": "src/platform-server.js",