@bongos/core 1.19.679 → 1.19.681

package/tests/upgrade.mjs

@@ -1091,6 +1091,265 @@ t('runUpgrade: refuses BEFORE the pin moves when the instance names no database'
   assert.match(readFileSync(join(dir, 'package.json'), 'utf8'), /"@bongos\/core": "1\.0\.0"/);
 });
 
+// ---- task 1002884: an upgrade may not declare success it has not proven -----
+// The 2026-08-11 cloudbongos.com incident, as executable tests. The sweep's `systemctl restart`
+// failed with "Interactive authentication required", upgrade.js logged it as a warning and
+// carried on, the health poll hit the STILL-RUNNING old process and got 200, and the run
+// reported "✓ upgrade complete — core 1.19.13 → 1.19.56" while live served 1.19.13.
+// Both halves are covered: a failed restart must be fatal, and a served version that did not
+// change must be caught even when the restart reported success.
+
+// A run shim whose `systemctl restart` fails, optionally letting `sudo -n systemctl restart` win.
+function restartFailingRun(calls, { dir, sudoWorks = false } = {}) {
+  const base = pinAwareRun(calls, { dir });
+  return (cmd, args = []) => {
+    if (cmd === 'systemctl' && args[0] === 'restart') { calls.push(`systemctl ${args.join(' ')}`); return { status: 1 }; }
+    if (cmd === 'sudo') { calls.push(`sudo ${args.join(' ')}`); return { status: sudoWorks ? 0 : 1 }; }
+    return base(cmd, args);
+  };
+}
+
+// A /version endpoint that reports whatever the live process is "serving".
+function versionFetch(serving, { healthOk = true } = {}) {
+  return async (url) => {
+    if (String(url).endsWith('/version')) return { ok: true, status: 200, json: async () => ({ coreVersion: serving }) };
+    return { ok: healthOk, status: healthOk ? 200 : 503 };
+  };
+}
+
+t('restartService: a non-root systemctl failure retries through `sudo -n` and can succeed there', () => {
+  const calls = [];
+  const run = (cmd, args = []) => { calls.push(`${cmd} ${args.join(' ')}`); return { status: cmd === 'sudo' ? 0 : 1 }; };
+  const r = u.restartService({ service: 'demo.service' }, run, { isRoot: false });
+  assert.equal(r.ok, true, 'the sudo -n retry restarted the service');
+  assert.equal(r.triedSudo, true);
+  assert.equal(r.directCode, 1, 'the direct attempt is reported alongside');
+  assert.deepEqual(calls, ['systemctl restart demo.service', 'sudo -n systemctl restart demo.service']);
+});
+
+t('restartService: as root there is no sudo retry (nothing to escalate to)', () => {
+  const calls = [];
+  const run = (cmd, args = []) => { calls.push(`${cmd} ${args.join(' ')}`); return { status: 1 }; };
+  const r = u.restartService({ service: 'demo.service' }, run, { isRoot: true });
+  assert.equal(r.ok, false);
+  assert.ok(!r.triedSudo, 'root does not shell out to sudo');
+  assert.deepEqual(calls, ['systemctl restart demo.service']);
+});
+
+t('restartService: an explicit --restart-cmd is never escalated (the operator owns that contract)', () => {
+  const calls = [];
+  const run = (cmd, args = []) => { calls.push(`${cmd} ${args.join(' ')}`); return { status: 1 }; };
+  const r = u.restartService({ restartCmd: 'deploy restart' }, run, { isRoot: false });
+  assert.equal(r.ok, false);
+  assert.deepEqual(calls, ['deploy restart'], 'no sudo retry for an explicit command');
+});
+
+t('the 2026-08-11 incident: a failed restart FAILS the bump even though /healthz answers 200', async () => {
+  const src = mkdtempSync(join(tmpdir(), 'src-'));
+  writeFileSync(join(src, 'bongos-core-1.15.0.tgz'), 'TGZ');
+  const dir = scratchConsumer({ pinned: '1.14.1', installed: '1.14.1' });
+  const calls = [], queries = [];
+  const res = await u.runUpgrade(
+    { to: '1.15.0', from: join(src, 'bongos-core-1.15.0.tgz'), instance: dir, service: 'demo.service', healthUrl: 'http://x/healthz' },
+    {
+      log: () => {}, err: () => {},
+      run: restartFailingRun(calls, { dir, sudoWorks: false }),
+      isRoot: false,
+      materialize: () => ({ skills: 1 }), regenerateApiArtifacts: () => ({ ok: true, ran: [] }),
+      databaseUrl: 'postgres://x/y', pg: recordingPg(queries),
+      // the old process is still up and healthy — the exact false-pass this task exists to kill
+      fetch: async () => ({ ok: true, status: 200 }),
+      sleep: async () => {}, now: () => 0, healthBudgetMs: 0,
+    }
+  );
+  assert.equal(res.ok, false, 'a failed restart is not a successful upgrade');
+  assert.equal(res.restartOk, false);
+  assert.match(res.error, /restart failed/i);
+  assert.match(res.error, /still running the previous core/i);
+  assert.equal(res.rolledBack, true, 'the bump reverts rather than leaving disk ahead of the process');
+  assert.equal(u.readPinnedCoreVersion(dir), '1.14.1', 'pin reverted');
+  assert.equal(queries.length, 1, 'no phantom "upgraded" ledger row');
+  assert.match(queries[0].params[3], /^rolled_back: restart failed/);
+  assert.ok(calls.includes('sudo -n systemctl restart demo.service'), 'it tried to escalate before giving up');
+});
+
+t('a restart that fails directly but succeeds under sudo -n carries the bump through', async () => {
+  const src = mkdtempSync(join(tmpdir(), 'src-'));
+  writeFileSync(join(src, 'bongos-core-1.15.0.tgz'), 'TGZ');
+  const dir = scratchConsumer({ pinned: '1.14.1', installed: '1.14.1' });
+  const calls = [], queries = [];
+  const res = await u.runUpgrade(
+    { to: '1.15.0', from: join(src, 'bongos-core-1.15.0.tgz'), instance: dir, service: 'demo.service', healthUrl: 'http://x/healthz' },
+    {
+      log: () => {}, err: () => {},
+      run: restartFailingRun(calls, { dir, sudoWorks: true }),
+      isRoot: false,
+      materialize: () => ({ skills: 1 }), regenerateApiArtifacts: () => ({ ok: true, ran: [] }),
+      databaseUrl: 'postgres://x/y', pg: recordingPg(queries),
+      fetch: versionFetch('1.15.0'),
+      sleep: async () => {}, now: () => 0, healthBudgetMs: 0,
+    }
+  );
+  assert.equal(res.ok, true, 'escalating is a success, not a warning');
+  assert.equal(res.servedOk, true);
+  assert.equal(res.servedVersion, '1.15.0');
+  assert.equal(queries.length, 1);
+  assert.ok(!/rolled_back/.test(queries[0].params[3] || ''), 'a real upgraded row, not a reversal');
+});
+
+t('served-version mismatch: restart reports success but the OLD core is still serving → fail + rollback', async () => {
+  const src = mkdtempSync(join(tmpdir(), 'src-'));
+  writeFileSync(join(src, 'bongos-core-1.15.0.tgz'), 'TGZ');
+  const dir = scratchConsumer({ pinned: '1.14.1', installed: '1.14.1' });
+  const calls = [], queries = [];
+  const res = await u.runUpgrade(
+    { to: '1.15.0', from: join(src, 'bongos-core-1.15.0.tgz'), instance: dir, service: 'demo.service', healthUrl: 'http://x/healthz' },
+    {
+      log: () => {}, err: () => {},
+      run: pinAwareRun(calls, { dir }), // systemctl exits 0 — nothing else can catch this
+      materialize: () => ({ skills: 1 }), regenerateApiArtifacts: () => ({ ok: true, ran: [] }),
+      databaseUrl: 'postgres://x/y', pg: recordingPg(queries),
+      fetch: versionFetch('1.14.1'), // the process never swapped
+      sleep: async () => {}, now: () => 0, healthBudgetMs: 0,
+    }
+  );
+  assert.equal(res.ok, false, 'disk-and-health agreement is not proof the new core is serving');
+  assert.equal(res.servedOk, false);
+  assert.equal(res.servedVersion, '1.14.1');
+  assert.equal(res.rolledBack, true);
+  assert.equal(queries.length, 1, 'no phantom "upgraded" row');
+  assert.match(queries[0].params[3], /^rolled_back: served core version did not change/);
+});
+
+t('served-version match: the bump is confirmed against the running process', async () => {
+  const src = mkdtempSync(join(tmpdir(), 'src-'));
+  writeFileSync(join(src, 'bongos-core-1.15.0.tgz'), 'TGZ');
+  const dir = scratchConsumer({ pinned: '1.14.1', installed: '1.14.1' });
+  const urls = [];
+  const res = await u.runUpgrade(
+    { to: '1.15.0', from: join(src, 'bongos-core-1.15.0.tgz'), instance: dir, service: 'demo.service', healthUrl: 'http://x/healthz' },
+    {
+      log: () => {}, err: () => {},
+      run: pinAwareRun([], { dir }),
+      materialize: () => ({ skills: 1 }), regenerateApiArtifacts: () => ({ ok: true, ran: [] }),
+      fetch: async (url) => { urls.push(String(url)); return versionFetch('1.15.0')(url); },
+      sleep: async () => {}, now: () => 0, healthBudgetMs: 0,
+    }
+  );
+  assert.equal(res.ok, true);
+  assert.equal(res.servedOk, true);
+  assert.ok(urls.includes('http://x/version'), 'the version URL is derived from the health URL origin');
+});
+
+t('an unreadable DERIVED version endpoint warns but does not fail the bump (no regression)', async () => {
+  const src = mkdtempSync(join(tmpdir(), 'src-'));
+  writeFileSync(join(src, 'bongos-core-1.15.0.tgz'), 'TGZ');
+  const dir = scratchConsumer({ pinned: '1.14.1', installed: '1.14.1' });
+  const warnings = [];
+  const res = await u.runUpgrade(
+    { to: '1.15.0', from: join(src, 'bongos-core-1.15.0.tgz'), instance: dir, service: 'demo.service', healthUrl: 'http://x/healthz' },
+    {
+      log: () => {}, err: (m) => warnings.push(String(m)),
+      run: pinAwareRun([], { dir }),
+      materialize: () => ({ skills: 1 }), regenerateApiArtifacts: () => ({ ok: true, ran: [] }),
+      fetch: async (url) => (String(url).endsWith('/version') ? { ok: false, status: 404 } : { ok: true, status: 200 }),
+      sleep: async () => {}, now: () => 0, healthBudgetMs: 0,
+    }
+  );
+  assert.equal(res.ok, true, 'an instance that does not serve /version can still be upgraded');
+  assert.equal(res.servedOk, null, 'unconfirmed is neither pass nor fail');
+  assert.ok(warnings.some((w) => /NOT confirmed against the running process/.test(w)), 'but it says so loudly');
+});
+
+t('an unreadable EXPLICIT --version-url fails the bump (proof was asked for)', async () => {
+  const src = mkdtempSync(join(tmpdir(), 'src-'));
+  writeFileSync(join(src, 'bongos-core-1.15.0.tgz'), 'TGZ');
+  const dir = scratchConsumer({ pinned: '1.14.1', installed: '1.14.1' });
+  const res = await u.runUpgrade(
+    { to: '1.15.0', from: join(src, 'bongos-core-1.15.0.tgz'), instance: dir, service: 'demo.service', healthUrl: 'http://x/healthz', versionUrl: 'http://x/v', rollbackOnFailure: false },
+    {
+      log: () => {}, err: () => {},
+      run: pinAwareRun([], { dir }),
+      materialize: () => ({ skills: 1 }), regenerateApiArtifacts: () => ({ ok: true, ran: [] }),
+      fetch: async (url) => (String(url) === 'http://x/v' ? { ok: false, status: 500 } : { ok: true, status: 200 }),
+      sleep: async () => {}, now: () => 0, healthBudgetMs: 0,
+    }
+  );
+  assert.equal(res.ok, false);
+  assert.equal(res.servedOk, false);
+});
+
+t('--no-health-check also opts out of the served-version read-back (one documented escape hatch)', async () => {
+  const src = mkdtempSync(join(tmpdir(), 'src-'));
+  writeFileSync(join(src, 'bongos-core-1.15.0.tgz'), 'TGZ');
+  const dir = scratchConsumer({ pinned: '1.14.1', installed: '1.14.1' });
+  let fetched = false;
+  const res = await u.runUpgrade(
+    { to: '1.15.0', from: join(src, 'bongos-core-1.15.0.tgz'), instance: dir, service: 'demo.service', healthCheck: false },
+    {
+      log: () => {}, err: () => {},
+      run: pinAwareRun([], { dir }),
+      materialize: () => ({ skills: 1 }), regenerateApiArtifacts: () => ({ ok: true, ran: [] }),
+      fetch: async () => { fetched = true; return { ok: true, status: 200 }; },
+    }
+  );
+  assert.equal(res.ok, true);
+  assert.equal(fetched, false, 'nothing is polled when the confirmation is explicitly waived');
+});
+
+t('the ROLLBACK restart escalates too — the restart after a failure is the one that matters', async () => {
+  const src = mkdtempSync(join(tmpdir(), 'src-'));
+  writeFileSync(join(src, 'bongos-core-1.15.0.tgz'), 'TGZ');
+  const dir = scratchConsumer({ pinned: '1.14.1', installed: '1.14.1' });
+  const calls = [], queries = [];
+  // The forward restart works; the health check then fails, and the rollback's restart is the
+  // one that needs sudo. Without deps reaching rollback's restartService there is no seam to
+  // assert this on, and the escalation that runs after a failure would go untested.
+  let restarts = 0;
+  const base = pinAwareRun(calls, { dir });
+  const run = (cmd, args = []) => {
+    if (cmd === 'systemctl' && args[0] === 'restart') {
+      restarts++;
+      calls.push(`systemctl ${args.join(' ')}`);
+      return { status: restarts === 1 ? 0 : 1 }; // forward ok, rollback needs escalation
+    }
+    if (cmd === 'sudo') { calls.push(`sudo ${args.join(' ')}`); return { status: 0 }; }
+    return base(cmd, args);
+  };
+  const res = await u.runUpgrade(
+    { to: '1.15.0', from: join(src, 'bongos-core-1.15.0.tgz'), instance: dir, service: 'demo.service', healthUrl: 'http://x/healthz' },
+    {
+      log: () => {}, err: () => {},
+      run, isRoot: false,
+      materialize: () => ({ skills: 1 }), regenerateApiArtifacts: () => ({ ok: true, ran: [] }),
+      databaseUrl: 'postgres://x/y', pg: recordingPg(queries),
+      fetch: async () => ({ ok: false, status: 503 }), // health fails forward AND after rollback
+      sleep: async () => {}, now: () => 0, healthBudgetMs: 0,
+    }
+  );
+  assert.equal(res.ok, false);
+  assert.equal(res.rolledBack, true);
+  assert.equal(res.rollback.restartOk, true, 'the rollback restart succeeded via sudo -n');
+  assert.ok(calls.includes('sudo -n systemctl restart demo.service'), 'the rollback path escalated');
+  assert.equal(u.readInstalledCoreVersion(dir), '1.14.1', 'the previous core is back on disk');
+});
+
+t('deriveVersionUrl: /version on the health URL origin, query + fragment dropped', () => {
+  assert.equal(u.deriveVersionUrl('http://127.0.0.1:3002/healthz'), 'http://127.0.0.1:3002/version');
+  assert.equal(u.deriveVersionUrl('https://demo.example.com/healthz?probe=1#x'), 'https://demo.example.com/version');
+  assert.equal(u.deriveVersionUrl('not a url'), null);
+});
+
+t('pollServedVersion: gives up rather than spinning when the clock never advances', async () => {
+  const r = await u.pollServedVersion('http://x/version', {
+    fetch: async () => ({ ok: true, status: 200, json: async () => ({}) }), // never carries a coreVersion
+    sleep: async () => {}, now: () => 0, versionBudgetMs: 1_000, versionMaxAttempts: 3,
+  });
+  assert.equal(r.ok, false);
+  assert.equal(r.attempts, 3, 'the attempt cap bounds a frozen clock');
+  assert.match(r.error, /no coreVersion/);
+});
+
 await Promise.all(pending);
 console.log(`\nupgrade: ${passed} passed, ${failed} failed`);
 process.exit(failed ? 1 : 0);