@bongos/core 1.19.835 → 1.19.837

package/tests/rebase_gate_unprovable.mjs

@@ -0,0 +1,134 @@
+// tests/rebase_gate_unprovable.mjs — a strand the builder CANNOT clear must stop
+// blocking their queue without stopping being reported (task 1003294).
+//
+// THE BUG. publish-reconciler proves a land by looking for a branch / published tip
+// in the CORE repo. A task that legitimately delivers into the INSTANCE repo (task
+// 1003253 — a real, complete, deployed commit on the instance host) can never be
+// proven here, so the sweep writes needs_rebase_reason='strand:no_branch_no_tip'.
+// ADR 0120 part 3 then refuses that builder EVERY new claim until they land or
+// release it — and they can do neither, because there is nothing in this repository
+// to land. The recorded remedy was a hand-run SQL UPDATE on the droplet.
+//
+// THE SHAPE OF THE FIX, and what these tests are really protecting: the gate and the
+// SURFACING were one function. "Stop blocking" must not become "stop telling them",
+// so the fix SPLITS them — gatingRebaseTasksForBuilder may release a strand past the
+// grace, needsRebaseTasksForBuilder keeps answering in full for me.js / the hall /
+// the CLI. Re-collapsing the two is the regression this file exists to catch.
+//
+// SQL SEMANTICS ARE NOT TESTED HERE — they need a real Postgres and live in
+// tests/claim_gate_rebase.mjs (INTEGRATION set). What is testable DB-free is the
+// wiring, the parameters, and the NULL-handling SHAPE of the predicate, and a
+// fail-open in that shape is the dangerous mistake, so it is asserted directly.
+//
+// Run:  node tests/rebase_gate_unprovable.mjs
+
+import { strict as assert } from 'node:assert';
+import { readFileSync } from 'node:fs';
+import path from 'node:path';
+import { fileURLToPath } from 'node:url';
+import { createRequire } from 'node:module';
+import { makeRunner } from './helpers.mjs';
+
+const require = createRequire(import.meta.url);
+const { test, summary } = makeRunner();
+const REPO_ROOT = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..');
+const read = (rel) => readFileSync(path.join(REPO_ROOT, rel), 'utf8');
+
+const shipMod = require('../modules/lifecycle/db-ship.js');
+const CLAIMS = read('modules/lifecycle/db-claims.js');
+const SHIP = read('modules/lifecycle/db-ship.js');
+const RECON = read('modules/lifecycle/publish-reconciler.js');
+
+// A client that records what it was asked, so the query can be inspected without a DB.
+const spyClient = () => {
+  const seen = [];
+  return { seen, query: async (sql, params) => { seen.push({ sql, params }); return { rows: [] }; } };
+};
+
+await test('the CLAIM GATE asks the gating query, not the surfacing one', async () => {
+  assert.match(CLAIMS, /gatingRebaseTasksForBuilder\(builderId, client\)/,
+    'claimTask must consult the gating subset');
+  assert.doesNotMatch(CLAIMS, /needsRebaseTasksForBuilder\(/,
+    'the gate must NOT use the unfiltered surfacing query — that is the bug, restored');
+});
+
+await test('the SURFACING query still returns everything — no exclusion at all', () => {
+  const body = SHIP.slice(
+    SHIP.indexOf('async function needsRebaseTasksForBuilder'),
+    SHIP.indexOf('// The ship gate (task 1002483')
+  );
+  assert.ok(body.length > 0, 'needsRebaseTasksForBuilder must still exist');
+  assert.doesNotMatch(body, /interval '1 millisecond'/,
+    'the surfacing query must not inherit the gate grace — the builder still sees the strand');
+  assert.doesNotMatch(body, /no_branch_no_tip/,
+    'the surfacing query must not filter on the strand reason');
+});
+
+await test('the gating query is parameterised with the grace and the reason', async () => {
+  const c = spyClient();
+  await shipMod.gatingRebaseTasksForBuilder(4242, c);
+  assert.equal(c.seen.length, 1);
+  const { sql, params } = c.seen[0];
+  assert.equal(params[0], 4242, 'builder id');
+  assert.equal(params[1], shipMod.UNPROVABLE_GATE_GRACE_MS, 'the grace is passed, not inlined');
+  assert.equal(params[2], 'strand:no_branch_no_tip',
+    'the reason is passed in the SPELLING announceStrand stores (strand:<reason>)');
+  assert.match(sql, /needs_rebase = true AND status = 'confirmed' AND shipped_by = \$1/,
+    'it must still be the same set of owed rebases, only narrowed');
+});
+
+await test('an explicit grace overrides the default — the window is tunable', async () => {
+  const c = spyClient();
+  await shipMod.gatingRebaseTasksForBuilder(1, c, 5_000);
+  assert.equal(c.seen[0].params[1], 5_000);
+});
+
+await test('a negative or junk grace degrades to 0, never to a negative interval', async () => {
+  // A negative interval would make `needs_rebase_at < now() + X` true for FRESH
+  // strands — releasing the gate immediately, the exact inversion of the intent.
+  for (const bad of [-1, NaN, undefined, 'nonsense']) {
+    const c = spyClient();
+    await shipMod.gatingRebaseTasksForBuilder(1, c, bad);
+    assert.ok(c.seen[0].params[1] >= 0, `grace ${String(bad)} must not go negative`);
+  }
+});
+
+await test('the predicate is FAIL-CLOSED on NULLs — an unknown keeps the gate', () => {
+  const body = SHIP.slice(SHIP.indexOf('async function gatingRebaseTasksForBuilder'));
+  const q = body.slice(0, body.indexOf('return rows;'));
+  // A bare `needs_rebase_reason = $3` yields NULL for a NULL reason, and
+  // NOT(NULL AND true AND true) is NULL, which drops the row from the WHERE —
+  // releasing the gate for a row we know nothing about. COALESCE removes the NULL.
+  assert.match(q, /COALESCE\(needs_rebase_reason, ''\) = \$3/,
+    'the reason comparison must not be able to produce NULL');
+  assert.match(q, /needs_rebase_at IS NOT NULL/,
+    'a NULL timestamp must not participate in the age comparison');
+  assert.match(q, /AND NOT \(/, 'the exclusion is expressed as NOT over non-NULL operands');
+});
+
+await test('the reconciler ESCALATES past the grace instead of going quiet', () => {
+  assert.match(RECON, /UNPROVABLE_GATE_GRACE_MS/,
+    'the sweep must use the SAME constant as the gate, not a second number that can drift');
+  assert.match(RECON, /require\('\.\/db-ship\.js'\)/, 'imported rather than restated');
+  assert.match(RECON, /UNPROVABLE task \$\{r\.task_id\}/,
+    'an ungated strand must still be named on every sweep');
+  assert.match(RECON, /still confirmed and still unlanded/,
+    'the log must say this is not a resolution');
+});
+
+await test('releasing the gate does NOT clear the flag — the row stays owed', () => {
+  // The fix must not touch needs_rebase/needs_rebase_reason. If a future change
+  // "tidied up" by clearing the flag past the grace, the strand would vanish from
+  // every surfacing and the task would silently sit confirmed forever.
+  const body = SHIP.slice(SHIP.indexOf('async function gatingRebaseTasksForBuilder'));
+  const q = body.slice(0, body.indexOf('return rows;'));
+  assert.doesNotMatch(q, /UPDATE|SET /, 'the gating query must be a pure read');
+});
+
+await test('the grace is bounded and sane by default', () => {
+  assert.ok(shipMod.UNPROVABLE_GATE_GRACE_MS > 0, 'a zero default would never gate at all');
+  assert.ok(shipMod.UNPROVABLE_GATE_GRACE_MS >= 60 * 60_000,
+    'too short and case (a) — the builder who simply never pushed — escapes the gate that exists for them');
+});
+
+summary();

package/tests/runner_drift.mjs

@@ -0,0 +1,173 @@
+// tests/runner_drift.mjs — the intent-runner drift gate (task 1002722).
+//
+// The incident: for weeks the prod intent runners ran core 1.19.2 from a retired clone while
+// the instance served a current core, so an owner's domain-attach click executed as a
+// TEARDOWN (task 1002716). The hotfix repointed the units with systemd drop-ins. These cases
+// pin the reading of a unit the way systemd reads it (a drop-in overrides the base file) and
+// the version comparison, against a fake filesystem shaped like the live box.
+//
+// Run:  node tests/runner_drift.mjs
+
+import { strict as assert } from 'node:assert';
+import { readFileSync } from 'node:fs';
+import { effectiveUnit, runnerScriptPath, coreVersionFor, installedCoreVersion, assessRunnerUnit, parseUnitList, runnerBanner } from '../scripts/gds/runner-drift.js';
+
+let passed = 0, failed = 0;
+function t(name, fn) {
+  try { fn(); passed++; console.log(`  PASS  ${name}`); }
+  catch (e) { failed++; console.log(`  FAIL  ${name}\n        ${e.message}`); }
+}
+
+// The live selfhost unit as `systemctl cat` prints it (2026-09-17): base file + drop-in.
+const LIVE_CAT = `# /etc/systemd/system/selfhost-provision-intent-runner.service
+[Service]
+Type=oneshot
+WorkingDirectory=/home/lars/cloudbongos
+EnvironmentFile=/etc/cloudbongos/box.env
+ExecStart=/usr/bin/env bash infra/core-node.sh scripts/gds/provision.js run-intents --apply
+
+# /etc/systemd/system/selfhost-provision-intent-runner.service.d/override.conf
+[Service]
+WorkingDirectory=/home/lars/cloudbongos-instance
+ExecStart=
+ExecStart=/usr/bin/node node_modules/@bongos/core/scripts/gds/provision.js run-intents --apply
+`;
+
+// A fake fs: path → file body. Only readFileSync + existsSync are used.
+function fakeFs(files) {
+  return {
+    existsSync: (p) => Object.prototype.hasOwnProperty.call(files, p),
+    readFileSync: (p) => { if (!(p in files)) { const e = new Error(`ENOENT ${p}`); e.code = 'ENOENT'; throw e; } return files[p]; },
+  };
+}
+const pkg = (name, version) => JSON.stringify({ name, version });
+const INSTANCE = '/home/lars/cloudbongos-instance';
+const boxFs = (installed, runnerPkg = installed) => fakeFs({
+  [`${INSTANCE}/node_modules/@bongos/core/package.json`]: pkg('@bongos/core', installed),
+  [`${INSTANCE}/package.json`]: pkg('cloudbongos-instance', '0.0.1'),
+  ...(runnerPkg !== installed ? { '/opt/other/node_modules/@bongos/core/package.json': pkg('@bongos/core', runnerPkg) } : {}),
+  '/home/lars/cloudbongos/src/module-api.js': "const CORE_VERSION = '1.19.2'; // old clone\n",
+  '/home/lars/cloudbongos/package.json': pkg('cloud-bongos', '0.0.2'),
+});
+
+t('effectiveUnit: a drop-in WorkingDirectory wins, and an empty ExecStart= resets the base one', () => {
+  assert.deepEqual(effectiveUnit(LIVE_CAT), {
+    workingDirectory: INSTANCE,
+    execStarts: ['/usr/bin/node node_modules/@bongos/core/scripts/gds/provision.js run-intents --apply'],
+  });
+});
+
+t('effectiveUnit: with no drop-in the base file stands (the pre-hotfix shape)', () => {
+  const base = LIVE_CAT.split('# /etc/systemd/system/selfhost-provision-intent-runner.service.d')[0];
+  const u = effectiveUnit(base);
+  assert.equal(u.workingDirectory, '/home/lars/cloudbongos');
+  assert.equal(u.execStarts.length, 1);
+  assert.match(u.execStarts[0], /infra\/core-node\.sh/);
+});
+
+t('assessRunnerUnit: a drop-in that ADDS an ExecStart without clearing leaves the stale base command running — never ok', () => {
+  const cat = LIVE_CAT.replace('ExecStart=\nExecStart=/usr/bin/node', 'ExecStart=/usr/bin/node')
+    .replace('ExecStart=/usr/bin/env bash infra/core-node.sh scripts/gds/provision.js run-intents --apply', 'ExecStart=/usr/bin/node /home/lars/cloudbongos/scripts/gds/provision.js run-intents --apply');
+  assert.equal(effectiveUnit(cat).execStarts.length, 2);
+  const r = assessRunnerUnit({ name: 'x', catText: cat, instanceRoot: INSTANCE, fsImpl: boxFs('1.19.831') });
+  assert.equal(r.status, 'drift', r.reason);
+  assert.match(r.reason, /1\.19\.2/);
+});
+
+t('runnerScriptPath: node + a relative script resolves against WorkingDirectory; a wrapper resolves to null', () => {
+  assert.equal(runnerScriptPath({ workingDirectory: INSTANCE, execStart: '/usr/bin/node node_modules/@bongos/core/scripts/gds/provision.js run-intents --apply' }),
+    `${INSTANCE}/node_modules/@bongos/core/scripts/gds/provision.js`);
+  assert.equal(runnerScriptPath({ workingDirectory: '/w', execStart: '/usr/bin/env node /abs/core/scripts/gds/box.js run-intents' }), '/abs/core/scripts/gds/box.js');
+  assert.equal(runnerScriptPath({ workingDirectory: '/w', execStart: '-/usr/bin/node scripts/gds/box.js' }), '/w/scripts/gds/box.js', 'systemd exec prefixes are stripped');
+  assert.equal(runnerScriptPath({ workingDirectory: '/w', execStart: '/usr/bin/env NODE_ENV=production node --max-old-space-size=512 scripts/gds/box.js' }), '/w/scripts/gds/box.js', 'env assignments and node flags are skipped');
+  assert.equal(runnerScriptPath({ workingDirectory: '/w', execStart: '/usr/bin/env bash infra/core-node.sh scripts/gds/provision.js' }), null,
+    'a shell wrapper decides at runtime which core it loads — the gate must not guess');
+});
+
+t('coreVersionFor: an installed package reads its package.json; a source checkout reads CORE_VERSION', () => {
+  const fs = boxFs('1.19.831');
+  assert.equal(coreVersionFor(`${INSTANCE}/node_modules/@bongos/core/scripts/gds/provision.js`, { fsImpl: fs }), '1.19.831');
+  assert.equal(coreVersionFor('/home/lars/cloudbongos/scripts/gds/provision.js', { fsImpl: fs }), '1.19.2');
+  assert.equal(coreVersionFor('/nowhere/scripts/gds/provision.js', { fsImpl: fs }), null);
+});
+
+t('installedCoreVersion: reads <root>/node_modules/@bongos/core; null when absent', () => {
+  assert.equal(installedCoreVersion(INSTANCE, { fsImpl: boxFs('1.19.831') }), '1.19.831');
+  assert.equal(installedCoreVersion('/empty', { fsImpl: boxFs('1.19.831') }), null);
+});
+
+t('assessRunnerUnit: the live drop-in shape is ok', () => {
+  const r = assessRunnerUnit({ name: 'selfhost-provision-intent-runner.service', catText: LIVE_CAT, instanceRoot: INSTANCE, fsImpl: boxFs('1.19.831') });
+  assert.equal(r.status, 'ok', r.reason);
+  assert.equal(r.runnerVersion, '1.19.831');
+});
+
+t('assessRunnerUnit: a unit running a source clone on another core is DRIFT, naming both versions', () => {
+  const cat = LIVE_CAT.replace(/ExecStart=\/usr\/bin\/node node_modules[^\n]*/, 'ExecStart=/usr/bin/node /home/lars/cloudbongos/scripts/gds/provision.js run-intents --apply');
+  const r = assessRunnerUnit({ name: 'x.service', catText: cat, instanceRoot: INSTANCE, fsImpl: boxFs('1.19.831') });
+  assert.equal(r.status, 'drift');
+  assert.match(r.reason, /1\.19\.2/);
+  assert.match(r.reason, /1\.19\.831/);
+});
+
+t('assessRunnerUnit: a wrapper, or a version it cannot read, is UNRESOLVED — never reported as ok', () => {
+  const base = LIVE_CAT.split('# /etc/systemd/system/selfhost-provision-intent-runner.service.d')[0];
+  assert.equal(assessRunnerUnit({ name: 'x', catText: base, instanceRoot: INSTANCE, fsImpl: boxFs('1.19.831') }).status, 'unresolved');
+  assert.equal(assessRunnerUnit({ name: 'x', catText: LIVE_CAT, instanceRoot: '/empty', fsImpl: boxFs('1.19.831') }).status, 'unresolved');
+  assert.equal(assessRunnerUnit({ name: 'x', catText: '', instanceRoot: INSTANCE, fsImpl: boxFs('1.19.831') }).status, 'unresolved');
+});
+
+t('parseUnitList: plain list-units output → service names only', () => {
+  const out = 'cohost-provision-intent-runner.service loaded inactive dead Cloud Bongos — drain\nselfhost-provision-intent-runner.service loaded inactive dead x\n\n';
+  assert.deepEqual(parseUnitList(out), ['cohost-provision-intent-runner.service', 'selfhost-provision-intent-runner.service']);
+  assert.deepEqual(parseUnitList(''), []);
+});
+
+t('runnerBanner: names the runner core + path, and flags a differing instance core', () => {
+  assert.equal(runnerBanner({ runnerVersion: '1.19.831', runnerRoot: '/r', instanceVersion: '1.19.831', instanceRoot: '/i' }),
+    'runner core 1.19.831 from /r (instance core 1.19.831 at /i)');
+  assert.match(runnerBanner({ runnerVersion: '1.19.2', runnerRoot: '/r', instanceVersion: '1.19.831', instanceRoot: '/i' }), /^⚠ DRIFT: runner core 1\.19\.2 from \/r \(instance core 1\.19\.831 at \/i\)/);
+  assert.equal(runnerBanner({ runnerVersion: null, runnerRoot: '/r', instanceVersion: null, instanceRoot: '/i' }), 'runner core ? from /r (no installed core at /i)');
+});
+
+// box.js run-intents is the other runner the incident class covers — it opens with the same line.
+const { cmdRunIntents: boxRunIntents } = await import('../scripts/gds/box.js');
+{
+  const lines = [];
+  const origLog = console.log;
+  const origRoot = process.env.BONGOS_INSTANCE_ROOT;
+  process.env.BONGOS_INSTANCE_ROOT = '/no/such/instance';
+  console.log = (l) => lines.push(String(l));
+  try { await boxRunIntents({ query: async () => ({ rows: [] }) }, {}, null, { apply: false }); }
+  finally {
+    console.log = origLog;
+    if (origRoot === undefined) delete process.env.BONGOS_INSTANCE_ROOT; else process.env.BONGOS_INSTANCE_ROOT = origRoot;
+  }
+  t('box.js run-intents opens with the runner core version line', () => {
+    assert.match(lines[0], /^runner core \d+\.\d+\.\d+ from .+ \(no installed core at \/no\/such\/instance\)$/);
+  });
+}
+
+const { runnerDriftRows } = await import('../scripts/gds/doctor.js');
+
+t('doctor maps each verdict to a row: ok → ok, drift → FAIL with the fix, unresolved → warn', () => {
+  const rows = runnerDriftRows([
+    { name: 'a.service', status: 'ok', reason: 'runs core 1.19.831' },
+    { name: 'b.service', status: 'drift', reason: 'runs core 1.19.2 (/x) but the instance installed 1.19.831 at /i' },
+    { name: 'c.service', status: 'unresolved', reason: 'ExecStart does not run a .js script directly' },
+  ]);
+  assert.deepEqual(rows.map((r) => r.status), ['ok', 'fail', 'warn']);
+  assert.equal(rows[0].msg, 'a.service: runs core 1.19.831');
+  assert.match(rows[1].msg, /^b\.service: runs core 1\.19\.2 .* — repoint the unit's WorkingDirectory \+ ExecStart at the installed core/);
+  assert.equal(rows[2].msg, 'c.service: ExecStart does not run a .js script directly');
+  assert.deepEqual(runnerDriftRows(null), [], 'no systemd → no rows, so a laptop shows no section');
+  assert.deepEqual(runnerDriftRows([]), []);
+});
+
+t('doctor runs the runner-drift probe', () => {
+  const src = readFileSync(new URL('../scripts/gds/doctor.js', import.meta.url), 'utf8');
+  assert.match(src.slice(src.indexOf('require.main === module')), /checkRunnerDrift\(\)/);
+});
+
+console.log(`\nrunner_drift: ${passed} passed, ${failed} failed`);
+process.exit(failed ? 1 : 0);