@bongos/core 1.21.117 → 1.21.118

package/scripts/gds/provision-auto-upgrade.js

@@ -0,0 +1,139 @@
+'use strict';
+
+// scripts/gds/provision-auto-upgrade.js — the hub moves each hosted project's core on
+// its own, by that project's update rule (task 1004729, ADR 0381).
+//
+// WHY THIS EXISTS. Every runner tick, snapshotAllCoreVersions works out `recommended`
+// for every active cloud-host under its own rule (pinned / patch / minor, follow
+// released) — and then stopped. A move only happened when someone pressed the button on
+// /deploy, so a project on `patch` sat on 1.21.20 for 11 days while 1.21.69 was out
+// (toast, 2026-10). The rule already IS the owner's decision about taking updates; this
+// step acts on it, with the same two steps the button uses:
+//
+//   1. behind, not pinned, no open intent, target not held → queue a
+//      `core-upgrade-preflight` for `recommended` (the dry run; changes nothing);
+//   2. that preflight PASSED for that same target → queue the `core-upgrade` to it.
+//
+// Every intent goes through provisioning.enqueueIntent, exactly as the runner's own
+// reconcile does, so the control-plane guard and the single-open-intent slot apply, and
+// /deploy shows these moves like any other (requested_by 'runner:auto-upgrade').
+//
+// MINIMAL FAILURE HOLD. A passing preflight does not run migrations (toast's 1.21.115
+// passed its preflight and failed at migrate, intent 267), so without a hold this would
+// re-queue a target that already failed, every tick, forever. A target is held for an
+// instance when it already has, FOR THAT TARGET: a core-upgrade that ended (error, or
+// done but the box is still behind), a preflight intent that errored, or a stored
+// preflight verdict that refused. A newer release is a new target and is not held. The
+// richer hold — telling the project owner, releasing it — is task 1004730.
+//
+// Carved into its own file because provision.js and provision-core-upgrade.js sit at
+// the size ratchet. Best-effort: nothing here may fail the tick.
+
+const { resolveEnv } = require('../../src/instance-config');
+
+// At most this many automatic intents per tick, across ALL projects, so a release never
+// restarts every hosted project at once. The tick runs every few minutes; one is plenty.
+const AUTO_MOVES_PER_TICK = 1;
+
+const REQUESTED_BY = 'runner:auto-upgrade';
+const OPEN_STATES = new Set(['pending', 'running']);
+const OFF_VALUES = new Set(['off', '0', 'false', 'no', 'disabled']);
+
+/**
+ * The platform-wide off switch: `<PREFIX>_AUTO_CORE_UPGRADE` (BONGOS_AUTO_CORE_UPGRADE on
+ * a vanilla hub) in the runner's environment. Unset means ON — the owner asked for this
+ * behaviour (2026-10-14). off / 0 / false / no / disabled stops every automatic move; the
+ * per-project `pinned` rule stays the project-level off switch. PURE over `env`.
+ */
+function autoUpgradeEnabled(env = process.env) {
+  const v = resolveEnv('AUTO_CORE_UPGRADE', { env, warn: () => {}, warnPack: () => {} });
+  if (v === undefined || v === null || String(v).trim() === '') return true;
+  return !OFF_VALUES.has(String(v).trim().toLowerCase());
+}
+
+function parseVerdict(p) {
+  if (!p) return null;
+  if (typeof p === 'string') { try { return JSON.parse(p); } catch { return null; } }
+  return p;
+}
+
+/**
+ * What, if anything, to queue for one project. PURE.
+ * @param {object} inst     provisioning_instances row joined with its core-version snapshot
+ * @param {object[]} intents that instance's intents (open ones + core-upgrade history)
+ * @param {object} channelMod update-channel.js
+ * @returns {{ action: string, to: string } | { skip: string }}
+ */
+function decideAutoMove(inst, intents, channelMod) {
+  if (inst.hosting_shape !== 'cloud-host') return { skip: 'not a hosted project' }; // never the control-plane row
+  if (inst.status !== 'active') return { skip: 'not active' };
+  if (channelMod.normalizeChannel(inst.update_channel) === 'pinned') return { skip: 'pinned' };
+  const served = inst.served_version;
+  const to = inst.recommended;
+  if (!served || !to || channelMod.compareSemver(to, served) <= 0) return { skip: 'current' };
+  if (intents.some((i) => OPEN_STATES.has(i.state))) return { skip: 'open intent' };
+  const same = (i) => String(i.target_version || '') === String(to);
+  const held = intents.some((i) => same(i) && (
+    (i.action === 'core-upgrade' && (i.state === 'error' || i.state === 'done'))
+    || (i.action === 'core-upgrade-preflight' && i.state === 'error')));
+  const verdict = parseVerdict(inst.preflight);
+  const verdictForTarget = verdict && String(verdict.target || '') === String(to) ? verdict : null;
+  if (held || (verdictForTarget && verdictForTarget.ok !== true)) return { skip: `held: ${to} already failed here` };
+  if (verdictForTarget) return { action: 'core-upgrade', to };
+  return { action: 'core-upgrade-preflight', to };
+}
+
+/**
+ * Queue at most AUTO_MOVES_PER_TICK automatic core intents. Runs after the snapshot on an
+ * --apply tick. Never throws.
+ */
+async function autoUpgradeTick(deps) {
+  const { db, provisioning, log = () => {} } = deps;
+  if (!autoUpgradeEnabled(deps.env || process.env)) {
+    log('auto core-upgrade: off (AUTO_CORE_UPGRADE) — no automatic moves');
+    return { queued: 0, off: true };
+  }
+  const channelMod = deps.updateChannel || require('./update-channel.js');
+  let queued = 0;
+  try {
+    const { rows } = await db.query(
+      `SELECT i.id, i.slug, i.owner_builder_id, i.hosting_shape, i.status, i.update_channel,
+              s.served_version, s.recommended, s.preflight
+         FROM provisioning_instances i
+         JOIN provisioning_core_version_snapshot s ON s.instance_id = i.id
+        WHERE i.status = 'active' AND i.hosting_shape = 'cloud-host' AND s.recommended IS NOT NULL
+        ORDER BY i.id`);
+    if (!rows.length) return { queued };
+    const { rows: intents } = await db.query(
+      `SELECT instance_id, action, state, target_version FROM provisioning_intents
+        WHERE instance_id = ANY($1)
+          AND (state IN ('pending', 'running')
+               OR (action IN ('core-upgrade', 'core-upgrade-preflight') AND target_version IS NOT NULL))`,
+      [rows.map((r) => r.id)]);
+    for (const inst of rows) {
+      if (queued >= AUTO_MOVES_PER_TICK) break;
+      const mine = intents.filter((i) => String(i.instance_id) === String(inst.id));
+      const d = decideAutoMove(inst, mine, channelMod);
+      if (d.skip) { if (d.skip.startsWith('held')) log(`  auto core-upgrade: ${inst.slug} ${d.skip}`); continue; }
+      try {
+        const r = await provisioning.enqueueIntent(db, inst.id, d.action, REQUESTED_BY, d.to);
+        if (!r || !r.created) continue; // lost a race for the open slot — not spent
+        queued++;
+        log(`  auto core-upgrade: ${inst.slug} ${inst.served_version} → ${d.to} (${d.action} queued)`);
+        if (provisioning.recordEvent) {
+          await provisioning.recordEvent(db, {
+            instanceId: inst.id, ownerBuilderId: inst.owner_builder_id, event: d.action, actor: REQUESTED_BY,
+            detail: d.action === 'core-upgrade' ? `automatic core move to ${d.to} (update rule)` : `automatic pre-flight check for ${d.to} (update rule)`,
+          }).catch(() => {});
+        }
+      } catch (e) {
+        log(`  auto core-upgrade: ${inst.slug} not queued (${(e && e.message) || e})`);
+      }
+    }
+  } catch (e) {
+    log(`  auto core-upgrade skipped (${(e && e.message) || e})`);
+  }
+  return { queued };
+}
+
+module.exports = { AUTO_MOVES_PER_TICK, autoUpgradeEnabled, autoUpgradeTick };

package/scripts/gds/provision-repo.js

@@ -383,6 +383,64 @@ function dbRoleGrantsCmd(inst, { privileged = false } = {}) {
   return asPostgres(`psql -v ON_ERROR_STOP=1 -d ${db} -c '${sql}'`, privileged);
 }
 
+// Hand OWNERSHIP of everything in the schema to the instance role (task 1004735). The grants
+// above are not enough: GRANT ALL carries no right to ALTER a table or CREATE OR REPLACE a
+// function, which only the OWNER has. Later core moves migrate AS the instance role
+// (--migrate-as, task 1004512), so the first migration that changed an existing table failed
+// with "must be owner of table tasks" and the move rolled back — toast, 2026-10-14.
+//
+// Object by object, never REASSIGN OWNED BY postgres: that also reassigns every SHARED object
+// postgres owns — sibling projects' whole databases included. Skipped on purpose:
+//   • objects an EXTENSION owns (pg_depend deptype e) — not the project's, and ALTER errors;
+//   • sequences linked to a table column (deptype a/i) — they follow their table, and an
+//     ALTER SEQUENCE on one errors;
+//   • row types of tables and auto-made array types — they follow their table / element.
+// Idempotent: only objects not already owned by the role are touched, so a re-run is a no-op.
+//
+// Same quoting rule as the commands above: dollar-quoted literals, format(%I)/regclass for
+// identifiers — no single or double quote, so it rides psql -c '…' and bash -s unchanged. And
+// never the text $p$: that marks a password for the runner's log redaction (provision.mjs), which
+// is why kind lists are array literals and a procedure is the ELSE of f/w/a.
+// Run as postgres, in the instance's own database: provision.js step 2b (right after the
+// grants), and the migrate_not_owner remedy in wedge-remedy.js for projects provisioned before.
+function dbOwnershipSql(role, schema = 'public') {
+  if (!/^[A-Za-z0-9_-]{1,63}$/.test(String(role || ''))) throw new Error(`dbOwnershipSql: role ${JSON.stringify(role)} is not a plain role name`);
+  if (!/^[a-z_][a-z0-9_]{0,62}$/.test(String(schema))) throw new Error(`dbOwnershipSql: schema ${JSON.stringify(schema)} is not a plain schema name`);
+  const r = `$r$${role}$r$`;
+  const ns = `$s$${schema}$s$::regnamespace`;
+  const notExt = (cat, col, types = '$e$e$e$') =>
+    `NOT EXISTS (SELECT 1 FROM pg_depend d WHERE d.classid = $c$${cat}$c$::regclass AND d.objid = ${col} AND d.deptype IN (${types}))`;
+  const k = (x) => `$k$${x}$k$`;
+  const t = (x) => `$t$${x}$t$`;
+  const anyOf = (list) => '$k${' + list + '}$k$::char[]'; // an array literal, not $k$p$k$ — see the $p$ note above
+  return `DO $own$ DECLARE o record; BEGIN ` +
+    // tables, partitions, views, materialized views, foreign tables — their indexes, row types
+    // and column-owned sequences follow them
+    `FOR o IN SELECT c.oid::regclass AS obj, c.relkind FROM pg_class c WHERE c.relnamespace = ${ns} ` +
+    `AND c.relkind = ANY (${anyOf('r,p,v,m,f')}) AND c.relowner <> ${r}::regrole AND ${notExt('pg_class', 'c.oid')} LOOP ` +
+    `EXECUTE format($f$ALTER %s %s OWNER TO %I$f$, CASE o.relkind WHEN ${k('v')} THEN ${t('VIEW')} WHEN ${k('m')} THEN ${t('MATERIALIZED VIEW')} WHEN ${k('f')} THEN ${t('FOREIGN TABLE')} ELSE ${t('TABLE')} END, o.obj, ${r}); END LOOP; ` +
+    // free-standing sequences
+    `FOR o IN SELECT c.oid::regclass AS obj FROM pg_class c WHERE c.relnamespace = ${ns} AND c.relkind = ${k('S')} ` +
+    `AND c.relowner <> ${r}::regrole AND ${notExt('pg_class', 'c.oid', '$e$e$e$, $e$a$e$, $e$i$e$')} LOOP ` +
+    `EXECUTE format($f$ALTER SEQUENCE %s OWNER TO %I$f$, o.obj, ${r}); END LOOP; ` +
+    // functions, procedures, aggregates — what CREATE OR REPLACE needs to own
+    `FOR o IN SELECT p.oid::regprocedure AS obj, p.prokind FROM pg_proc p WHERE p.pronamespace = ${ns} ` +
+    `AND p.proowner <> ${r}::regrole AND ${notExt('pg_proc', 'p.oid')} LOOP ` +
+    `EXECUTE format($f$ALTER %s %s OWNER TO %I$f$, CASE o.prokind WHEN ${k('f')} THEN ${t('FUNCTION')} WHEN ${k('w')} THEN ${t('FUNCTION')} WHEN ${k('a')} THEN ${t('AGGREGATE')} ELSE ${t('PROCEDURE')} END, o.obj, ${r}); END LOOP; ` +
+    // free-standing types: enums, domains, ranges, and composite types made by CREATE TYPE
+    `FOR o IN SELECT y.oid::regtype AS obj, y.typtype FROM pg_type y WHERE y.typnamespace = ${ns} AND y.typowner <> ${r}::regrole ` +
+    `AND (y.typtype = ANY (${anyOf('e,d,r')}) OR (y.typtype = ${k('c')} AND EXISTS (SELECT 1 FROM pg_class c WHERE c.oid = y.typrelid AND c.relkind = ${k('c')}))) ` +
+    `AND ${notExt('pg_type', 'y.oid')} LOOP ` +
+    `EXECUTE format($f$ALTER %s %s OWNER TO %I$f$, CASE o.typtype WHEN ${k('d')} THEN ${t('DOMAIN')} ELSE ${t('TYPE')} END, o.obj, ${r}); END LOOP; ` +
+    `END $own$;`;
+}
+
+// The ownership handover as a command (task 1004735): provisioning runs it after the grants,
+// and the migrate_not_owner remedy runs it for a project provisioned before that. PURE.
+function dbOwnershipCmd(inst, { privileged = false } = {}) {
+  return asPostgres(`psql -v ON_ERROR_STOP=1 -d ${dbName(inst)} -c '${dbOwnershipSql(instanceDbRole(inst))}'`, privileged);
+}
+
 // ── STANDALONE scaffold (ADR 0108 / 0125, task 2055) ──────────────────────────
 // The MISSING first leg of the greenfield "one guided flow": provision.js used to
 // THROW if the instance repo wasn't already cloned. These generate + run the scaffold
@@ -1242,6 +1300,8 @@ module.exports = {
   dbRoleCmd,
   dbRoleCmdShown,
   dbRoleGrantsCmd,
+  dbOwnershipCmd,
+  dbOwnershipSql,
   deployKeyPath,
   deployKeyTitle,
   ensureInstanceUserCmd,

package/scripts/gds/provision.js

@@ -49,7 +49,7 @@ const crypto = require('node:crypto');
 
 const retry = require('./intent-retry.js'); // when a failed intent is tried again (task 1004447)
 const { CONFIG, MANIFEST_UA, softFailResult, MANIFEST_VERIFY_INTERVAL_MS, MANIFEST_VERIFY_TRIES, MAX_INTENT_ATTEMPTS, REPO_ROOT, coreVersionSafe, hasFlag, loadDeps, oauthSecret, provisionerBotEmail } = require('./provision-config.js');
-const { accountPreflight, appUser, alreadyScaffolded, buildCorePinRefreshCommit, coreCheckoutHasGit, dbCreateCmd, dbName, dbRoleCmd, dbRoleCmdShown, dbRoleGrantsCmd, ensureInstanceUserCmd, generateDbPassword, grantInstanceRepoReadCmd, instanceDbRole, instanceStateDir, instanceUser, deployKeyPath, deployKeyTitle, ensurePrivateRepoAccess, installedCorePackDir, installedCoreTarball, instanceInitSpec, instanceRepoRemote, migrateCmd, onboardMode, ownerLoginOf, parseTargetRef, planCorePinRefresh, refreshStandaloneCorePin, resolveOwnerGithubToken, resolveVendorableCoreTarball, safeVersionLabel, scaffoldStandaloneRepo, seedFirstVersionCmd, standaloneInstallCmd, standaloneMigrateCmd, standalonePullCmd, standaloneRegenDocsCmd, standaloneRoot } = require('./provision-repo.js');
+const { accountPreflight, appUser, alreadyScaffolded, buildCorePinRefreshCommit, coreCheckoutHasGit, dbCreateCmd, dbName, dbRoleCmd, dbRoleCmdShown, dbRoleGrantsCmd, dbOwnershipCmd, ensureInstanceUserCmd, generateDbPassword, grantInstanceRepoReadCmd, instanceDbRole, instanceStateDir, instanceUser, deployKeyPath, deployKeyTitle, ensurePrivateRepoAccess, installedCorePackDir, installedCoreTarball, instanceInitSpec, instanceRepoRemote, migrateCmd, onboardMode, ownerLoginOf, parseTargetRef, planCorePinRefresh, refreshStandaloneCorePin, resolveOwnerGithubToken, resolveVendorableCoreTarball, safeVersionLabel, scaffoldStandaloneRepo, seedFirstVersionCmd, standaloneInstallCmd, standaloneMigrateCmd, standalonePullCmd, standaloneRegenDocsCmd, standaloneRoot } = require('./provision-repo.js');
 const { backupScriptPath, backupService, backupServicePath, backupTimer, backupTimerPath, backupUnitName, envPrefixMismatch, instanceManifestCmd, originEnvVarsFor, serviceUnit, serviceUnitPath, settingsConsumed, settingsEnvVarsFor, renamedEnvCarry, upsertEnvVars, webEnvBody, webEnvPath } = require('./provision-units.js');
 const { caddyBlock, caddySnippetPath, checkCaddyWiring, checkDnsTokenScope, classifyZoneScope, dnsReleaseEnv, dnsUpsertEnv, federateInstance, federationHubOrigin, healthzCmd, identityCmd, identityVerdict, ingressPlaneGap, ingressPreflight, shouldFederate, zoneForDomain } = require('./provision-net.js');
 const { resolveProcessManager } = require('./provision-process.js');
@@ -463,6 +463,11 @@ async function provisionInstance(inst, deps) {
   // call site also suppresses stderr). The GRANTS carry no secret and log in full.
   boxExec(dbRoleCmd(inst, { password: dbPassword, privileged }), { shown: dbRoleCmdShown(inst) });
   boxExec(dbRoleGrantsCmd(inst, { privileged }));
+  // Grants are not ownership: later moves migrate AS this role and must ALTER what the migrate
+  // above created as postgres (task 1004735). Non-fatal on purpose — if it ever fails, the
+  // project still serves, and the first move that needs it applies it (migrate_not_owner).
+  const own = boxExec(dbOwnershipCmd(inst, { privileged }), { allowFail: true });
+  if (own && own.softFailed) log('  ⚠ [db] could not hand the schema to the instance role — its first core move that ALTERs a table will retry it (migrate_not_owner)');
 
   // ── step 3: config + per-instance secrets ─────────────────────────────────
   log('  [config] per-instance web.env');
@@ -1280,6 +1285,7 @@ async function cmdRunIntents(deps) {
     }
   }
   await snapshotAllCoreVersions(deps); // refresh what the deploy control reads, AFTER the drain (ADR 0293 D3/D6)
+  await require('./provision-auto-upgrade.js').autoUpgradeTick(deps); // then act on each hosted project's update rule: preflight, then move, one per tick (task 1004729, ADR 0381)
   const m = await drainOAuthManifests(deps); // same tick: convert + place any GitHub-App-Manifest creds awaiting the control plane
   await require('./provision-github-setup.js').verifyGithubAppSetups(deps).catch((e) => log(`  github-setup check failed (non-blocking): ${e.message}`)); // install + device flow, asked of GitHub with the app's key (task 1004649)
   log(`done — ran=${ran} ok=${ok} errors=${errors}; oauth-manifest converted=${m.converted} errors=${m.errors}`);

package/scripts/gds/run-unit-tests.js

@@ -184,6 +184,10 @@ const INTEGRATION = new Set([
   // the --integration lane never saw them either, because it runs only this set. A
   // test that cannot fail in either lane is a file that looks like a gate.
   'handle_machinery', 'hub_profile_leak', 'profile_fields',
+  // task 1004735: the schema ownership handover (provision-repo.js dbOwnershipSql) against a
+  // real engine — one DO block of catalog queries, so only Postgres can say it runs, what it
+  // reaches and that it leaves extension objects alone. Throwaway schema + role, dropped after.
+  'db_ownership_handover',
   // task 1002300 (R24): guilds against real SQL. The DB-free sibling
   // (guilds_api) pins the route layer and statement shapes; only a real engine
   // proves the one-owner partial index, the expiry CTE drained before its insert,

package/scripts/gds/upgrade-outcome.js

@@ -66,6 +66,9 @@ const REASON_CLASS = Object.freeze({
   // a migration's CHECK constraint is broken by rows the project already has (task 1004734): a
   // retry fails identically and no remedy may rewrite the owner's data, so a person decides.
   migrate_check_violation: 'unknown',
+  // a migration ALTERs an object the project's own account does not OWN — the tables were made
+  // by postgres at provisioning (task 1004735). Known fix: hand them over (wedge-remedy.js).
+  migrate_not_owner: 'known_wedge',
   restart: 'unknown',
   health: 'unknown',
   // the runner's one automatic fix did not clear it, or itself failed (task 1004448)
@@ -90,6 +93,7 @@ const UPGRADE_REASONS = [
   [/^npm install failed/, 'npm_install'],
   [/^integrity pin/, 'integrity_pin'],
   [/^migrate failed .*\bERROR: .*check constraint/, 'migrate_check_violation'], // upgrade-migrate.js migrateError (task 1004734)
+  [/^migrate failed .*\bmust be owner of /, 'migrate_not_owner'], // Postgres' own wording (task 1004735)
   [/^migrate failed/, 'migrate'],
   [/^restart failed/, 'restart'],
   [/^(health check failed after restart|served core version did not change)/, 'health'],

package/scripts/gds/wedge-remedy.js

@@ -14,6 +14,10 @@
 //                     knows is safe to delete — they are copies, and the newest are kept).
 //   schema_pending  — the project's database is behind the code it is running NOW. Fix:
 //                     migrate that database, with its own PGDATABASE, before the code moves.
+//   migrate_not_owner — a migration changed a table or function the project's own account
+//                     does not OWN: provisioning made them as postgres (task 1004735). Fix:
+//                     hand ownership of the project's schema to its account, as postgres.
+//                     Heals every project provisioned before provision-repo.js did this itself.
 // intent-retry.js retires every known_wedge at once, so on its own the failure only waited
 // for a person. This file runs the fix, records it as an event on the project, and re-pends
 // the move for ONE more try.
@@ -33,13 +37,13 @@
 // runner's own exec and db, so every fix is testable with an injected exec.
 
 const { CONFIG } = require('./provision-config.js');
-const { dbName, selfInstanceRoot, standaloneRoot } = require('./provision-repo.js');
+const { dbName, dbOwnershipCmd, instanceDbRole, selfInstanceRoot, standaloneRoot } = require('./provision-repo.js');
 const { migratePlan, CORE_MIGRATE_SCRIPT, ACCOUNT_RE } = require('./upgrade-migrate.js');
 const { isValidSlug } = require('../../modules/provisioning/provisioning.js');
 const outcome = require('./upgrade-outcome.js');
 const { DEFAULT_RETRY_DELAY_MS } = require('./intent-retry.js');
 
-const REMEDY_REASONS = Object.freeze(['dirty_tree', 'disk_full', 'schema_pending']);
+const REMEDY_REASONS = Object.freeze(['dirty_tree', 'disk_full', 'schema_pending', 'migrate_not_owner']);
 // The newest dumps a disk-full prune always leaves behind: a week of nightlies. The nightly
 // job's own retention is 30 days; this only ever cuts below that when the disk is full.
 const DUMP_KEEP_FLOOR = 7;
@@ -95,6 +99,13 @@ function planRemedy(inst, reason, o = {}) {
       describe: `pruned ${db}'s old database backups in ${backupDir}, keeping the newest ${DUMP_KEEP_FLOOR}`,
     };
   }
+  if (reason === 'migrate_not_owner') {
+    // A control plane migrates as its own app user, which created its tables — there is no
+    // separate project account to hand them to, so the snag cannot be this one there.
+    if (inst.hosting_shape === 'control-plane') return { refused: 'the control plane has no separate project account to hand its tables to' };
+    const role = instanceDbRole(inst);
+    return { reason, cwd: null, env: null, cmd: dbOwnershipCmd(inst, { privileged: o.privileged }), describe: `handed ownership of ${db}'s tables and functions to ${role}, the account its migrations run as` };
+  }
   // schema_pending — the project's OWN migrate command (a declared script wins, else the
   // core's migrate.sh: upgrade-migrate.js), aimed at its OWN database. migrate.sh defaults to
   // production when PGDATABASE is unset, so it is always set, never inherited.

package/src/module-api.js

@@ -75,7 +75,7 @@ const { responsibilityFor, ROLE_RESPONSIBILITIES } = require('./role-responsibil
 // MAJOR (see allowBoxScope below): passes the request through untouched.
 function deprecatedNoopMiddleware(_req, _res, next) { next(); }
 
-const CORE_VERSION = '1.21.117'; // CI auto-patch carrier (ADR 0161); changelog: docs/module-api-changelog.md
+const CORE_VERSION = '1.21.118'; // CI auto-patch carrier (ADR 0161); changelog: docs/module-api-changelog.md
 
 // A namespaced logger so a module's log lines are attributable + consistent.
 // Usage: const log = api.logger('discord'); log.info('mounted');

package/tests/core_auto_upgrade.mjs

@@ -0,0 +1,189 @@
+// tests/core_auto_upgrade.mjs — the hub moves each hosted project's core on its own,
+// by that project's update rule (task 1004729, ADR 0381).
+//
+// The runner tick used to compute `recommended` for every cloud-host and stop; a move
+// only happened when someone pressed the button. These tests drive the new step with a
+// fake db and a fake provisioning port and assert WHAT IT QUEUES:
+//   behind on `patch`            → a preflight for `recommended`
+//   preflight passed that target → the move to it
+//   pinned / open intent / held target / control-plane row → nothing
+//   more candidates than the pace → only AUTO_MOVES_PER_TICK queued
+//   the platform-wide off switch  → nothing at all
+//
+// Run:  node tests/core_auto_upgrade.mjs
+
+import { strict as assert } from 'node:assert';
+import { autoUpgradeTick, AUTO_MOVES_PER_TICK, autoUpgradeEnabled } from '../scripts/gds/provision-auto-upgrade.js';
+
+let passed = 0;
+let failed = 0;
+const t = async (name, fn) => {
+  try { await fn(); passed += 1; console.log(`  PASS  ${name}`); }
+  catch (e) { failed += 1; console.log(`  FAIL  ${name}\n        ${e.message}`); }
+};
+
+// One hosted project, its snapshot row joined on, as the candidate query returns it.
+function row(over = {}) {
+  return {
+    id: 52, slug: 'toast', owner_builder_id: 7, hosting_shape: 'cloud-host', status: 'active',
+    update_channel: 'patch', served_version: '1.21.20', recommended: '1.21.69', preflight: null,
+    ...over,
+  };
+}
+
+// A fake world: `rows` are candidates, `intents` the provisioning_intents history.
+function makeDeps({ rows = [row()], intents = [], env = {} } = {}) {
+  const queued = [];
+  const events = [];
+  const db = {
+    async query(sql, params) {
+      if (/FROM provisioning_instances/.test(sql)) return { rows };
+      if (/FROM provisioning_intents/.test(sql)) {
+        const ids = params[0].map(Number);
+        return { rows: intents.filter((i) => ids.includes(Number(i.instance_id))) };
+      }
+      throw new Error(`unexpected query: ${sql}`);
+    },
+  };
+  const provisioning = {
+    async enqueueIntent(_db, instanceId, action, requestedBy, target) {
+      const intent = { id: 900 + queued.length, instance_id: instanceId, action, requested_by: requestedBy, target_version: target, state: 'pending' };
+      queued.push(intent);
+      return { intent, created: true };
+    },
+    async recordEvent(_db, e) { events.push(e); },
+  };
+  return { db, provisioning, env, log: () => {}, queued, events };
+}
+
+console.log('core auto-upgrade tick (task 1004729)');
+
+await t('a patch tenant behind gets a preflight queued for recommended, attributed to the runner', async () => {
+  const d = makeDeps();
+  const r = await autoUpgradeTick(d);
+  assert.equal(d.queued.length, 1);
+  assert.equal(d.queued[0].action, 'core-upgrade-preflight');
+  assert.equal(d.queued[0].target_version, '1.21.69');
+  assert.equal(d.queued[0].instance_id, 52);
+  assert.equal(d.queued[0].requested_by, 'runner:auto-upgrade');
+  assert.equal(r.queued, 1);
+});
+
+await t('a preflight that PASSED for that target queues the move to it', async () => {
+  const d = makeDeps({ rows: [row({ preflight: { target: '1.21.69', ok: true } })],
+    intents: [{ instance_id: 52, action: 'core-upgrade-preflight', state: 'done', target_version: '1.21.69' }] });
+  await autoUpgradeTick(d);
+  assert.equal(d.queued.length, 1);
+  assert.equal(d.queued[0].action, 'core-upgrade');
+  assert.equal(d.queued[0].target_version, '1.21.69');
+});
+
+await t('a passed preflight for a DIFFERENT target re-asks for the current one', async () => {
+  const d = makeDeps({ rows: [row({ preflight: { target: '1.21.50', ok: true } })] });
+  await autoUpgradeTick(d);
+  assert.equal(d.queued[0].action, 'core-upgrade-preflight');
+  assert.equal(d.queued[0].target_version, '1.21.69');
+});
+
+await t('a pinned tenant gets nothing', async () => {
+  const d = makeDeps({ rows: [row({ update_channel: 'pinned' })] });
+  await autoUpgradeTick(d);
+  assert.equal(d.queued.length, 0);
+});
+
+await t('a tenant with an open intent gets nothing new', async () => {
+  for (const state of ['pending', 'running']) {
+    const d = makeDeps({ intents: [{ instance_id: 52, action: 'restart', state, target_version: null }] });
+    await autoUpgradeTick(d);
+    assert.equal(d.queued.length, 0, state);
+  }
+});
+
+await t('a target held by a failed move is not queued (retired core-upgrade)', async () => {
+  const d = makeDeps({ rows: [row({ preflight: { target: '1.21.69', ok: true } })],
+    intents: [{ instance_id: 52, action: 'core-upgrade', state: 'error', target_version: '1.21.69' }] });
+  await autoUpgradeTick(d);
+  assert.equal(d.queued.length, 0);
+});
+
+await t('a target held by a refused preflight is not queued', async () => {
+  const d = makeDeps({ rows: [row({ preflight: { target: '1.21.69', ok: false, error: 'migration would fail' } })] });
+  await autoUpgradeTick(d);
+  assert.equal(d.queued.length, 0);
+});
+
+await t('a target whose preflight intent itself errored is not re-asked', async () => {
+  const d = makeDeps({ intents: [{ instance_id: 52, action: 'core-upgrade-preflight', state: 'error', target_version: '1.21.69' }] });
+  await autoUpgradeTick(d);
+  assert.equal(d.queued.length, 0);
+});
+
+await t('a move that finished but left the box behind is not re-queued every tick', async () => {
+  const d = makeDeps({ rows: [row({ preflight: { target: '1.21.69', ok: true } })],
+    intents: [{ instance_id: 52, action: 'core-upgrade', state: 'done', target_version: '1.21.69' }] });
+  await autoUpgradeTick(d);
+  assert.equal(d.queued.length, 0);
+});
+
+await t('an older failed target does not hold a newer one', async () => {
+  const d = makeDeps({ intents: [{ instance_id: 52, action: 'core-upgrade', state: 'error', target_version: '1.21.60' }] });
+  await autoUpgradeTick(d);
+  assert.equal(d.queued.length, 1);
+  assert.equal(d.queued[0].target_version, '1.21.69');
+});
+
+await t('the control-plane row (and any non-cloud-host) gets nothing', async () => {
+  const d = makeDeps({ rows: [row({ id: 1, slug: 'hub', hosting_shape: 'control-plane' }), row({ id: 3, slug: 'ct', hosting_shape: 'co-tenant' })] });
+  await autoUpgradeTick(d);
+  assert.equal(d.queued.length, 0);
+});
+
+await t('nothing is queued when recommended is not newer than served', async () => {
+  for (const over of [{ recommended: null }, { recommended: '1.21.20' }, { recommended: '1.21.10' }, { served_version: null }]) {
+    const d = makeDeps({ rows: [row(over)] });
+    await autoUpgradeTick(d);
+    assert.equal(d.queued.length, 0, JSON.stringify(over));
+  }
+});
+
+await t('at most the pace limit is queued per tick', async () => {
+  assert.equal(AUTO_MOVES_PER_TICK, 1);
+  const d = makeDeps({ rows: [row({ id: 52 }), row({ id: 53, slug: 'b' }), row({ id: 54, slug: 'c' })] });
+  const r = await autoUpgradeTick(d);
+  assert.equal(d.queued.length, AUTO_MOVES_PER_TICK);
+  assert.equal(r.queued, AUTO_MOVES_PER_TICK);
+});
+
+await t('the pace counts only intents actually created (a lost race does not spend it)', async () => {
+  const d = makeDeps({ rows: [row({ id: 52 }), row({ id: 53, slug: 'b' })] });
+  const real = d.provisioning.enqueueIntent;
+  let first = true;
+  d.provisioning.enqueueIntent = async (...a) => {
+    if (first) { first = false; return { intent: { state: 'pending' }, created: false }; }
+    return real(...a);
+  };
+  await autoUpgradeTick(d);
+  assert.equal(d.queued.length, 1);
+  assert.equal(d.queued[0].instance_id, 53);
+});
+
+await t('the off switch stops every automatic move, and it defaults ON', async () => {
+  for (const v of ['off', '0', 'false', 'no', 'OFF']) {
+    const d = makeDeps({ env: { BONGOS_AUTO_CORE_UPGRADE: v } });
+    const r = await autoUpgradeTick(d);
+    assert.equal(d.queued.length, 0, v);
+    assert.equal(r.off, true);
+  }
+  assert.equal(autoUpgradeEnabled({}), true);
+  assert.equal(autoUpgradeEnabled({ BONGOS_AUTO_CORE_UPGRADE: 'on' }), true);
+});
+
+await t('a refusal from enqueueIntent (e.g. the control-plane guard) never fails the tick', async () => {
+  const d = makeDeps();
+  d.provisioning.enqueueIntent = async () => { throw new Error('refused'); };
+  const r = await autoUpgradeTick(d);
+  assert.equal(r.queued, 0);
+});
+
+console.log(`\n${passed} passed, ${failed} failed`);
+if (failed) process.exit(1);