@bongos/core 1.21.174 → 1.21.176

package/scripts/bongos/provision-core-upgrade.js

@@ -0,0 +1,1166 @@
+'use strict';
+
+// scripts/bongos/provision-core-upgrade.js — the owner's deploy door, CONTROL-PLANE half
+// (task 1003159, ADR 0293).
+//
+// The web tier queues a 'core-upgrade' intent naming a target version and gains no new
+// capability doing so (modules/provisioning/core-upgrade.js). THIS is the half that acts:
+// the runner drains that intent and drives the proven `bongos upgrade` path, with its
+// health gate, its served-version check and its auto-rollback (task 2149) intact.
+//
+// It is a DOOR ONTO THE PROVEN PATH, NOT A SECOND UPGRADE MECHANISM. Every step that
+// could get an upgrade wrong — install, migrate, pin, restart, verify, revert — already
+// lives in upgrade.js and is exercised by every hand-run bump. Re-implementing any of it
+// here would mean two paths that must stay in step, and the one used less often would be
+// the one that rots. So this file does exactly three things upgrade.js cannot do for
+// itself: decide whether the request is ALLOWED, decide whether the target is REAL, and
+// report what the box actually serves afterwards.
+//
+// WHY THE REGISTRY CHECK LIVES HERE AND NOT ON THE ROUTE (ADR 0293 D3). Answering "is
+// this version published" means `npm view`, which update-channel.js runs via spawnSync —
+// on the single-process web server that blocks every other request for the seconds npm
+// takes. Here it is a CLI process with nothing else to serve, and blocking is free. The
+// check is not optional tidiness: the release lane mints a git tag BEFORE the npm publish
+// completes (8–13 minutes, measured 2026-09-16 — learnings 1000263/1000266), so a door
+// that trusted the request would routinely queue a version that does not exist yet and
+// fail deep inside npm with `notarget`.
+//
+// Its own file because scripts/bongos/provision.js sits at the 1500-line oversized-file
+// ratchet, the same reason provision-net.js and provision-units.js were carved out.
+
+const crypto = require('crypto');
+const fs = require('fs');
+const os = require('os');
+const path = require('path');
+const { spawnSync } = require('child_process');
+const { CONFIG, MANIFEST_VERIFY_INTERVAL_MS, MANIFEST_VERIFY_TRIES, REPO_ROOT } = require('./provision-config.js');
+const { appUser, dbName, instanceUser, parseTargetRef, resolveOwnerGithubToken, selfInstanceRoot, standaloneRoot } = require('./provision-repo.js');
+const { ensurePinWriteKey, hasOwnAccount, pinKeyPath, pinSshCommand } = require('./provision-pin-key.js');
+const { replayPinCommit } = require('./provision-pin-replay.js'); // GitHub ahead → pin on top (task 1004733)
+const { isValidSlug } = require('../../modules/provisioning/provisioning.js');
+const { RELEASE_NOTES_FILE, RELEASE_NOTES_MAX_BYTES, parseChangeRecords, notesBetween } = require('./release-notes.js');
+const outcome = require('./upgrade-outcome.js'); // how far a failed move got, as a class (task 1004446)
+
+// The shapes whose box this runner can actually reach. Mirrors UPGRADE_SHAPES in the
+// web-tier half, and deliberately EXCLUDES 'dedicated': a dedicated droplet is stood up
+// by cloud-init and the runner has no post-hoc exec path to it, so queueing an upgrade
+// for one would be a request that is accepted and never lands. The route refuses the
+// same shape; this is the second gate, and it is the one that holds if a row's shape
+// changes between the request and the drain.
+const UPGRADE_SHAPES = Object.freeze(new Set(['co-tenant', 'cloud-host', 'control-plane']));
+
+// The strict three-number form a core version takes. Re-stated here rather than imported
+// from the web tier's core-upgrade.js: a module may not be required from scripts/ (ADR
+// 0083) and this side needs it to guard a SHELL interpolation, which is a stricter duty
+// than validating a request body.
+const CORE_VERSION_RE = /^[0-9]+\.[0-9]+\.[0-9]+$/;
+
+// The box's own answer to "what core am I running?" — loopback, on the instance's port.
+//
+// THE APEX CANNOT BE ASKED (ADR 0293 D6). It sits behind pre-launch basic_auth and its
+// HTML rides a CDN, so a public read can report a stale or gated answer with complete
+// confidence. 127.0.0.1 is the only address that is answering FROM the process whose
+// version is in question. This is the same reasoning healthzCmd already encodes, and the
+// DONE-WHEN task 1003156 established in so many words.
+function versionCmd(inst) {
+  return `curl -sS --max-time 5 http://127.0.0.1:${inst.port}/version`;
+}
+
+// Pull coreVersion out of whatever /version answers with.
+//
+// TOLERANT BY DESIGN, and the tolerance is load-bearing rather than lazy. This reads a
+// process that may be running a core OLDER than the one that wrote this parser — that is
+// the entire point of an upgrade — so the shape of its response is not something this
+// side gets to assume. A body it cannot read returns null, which the caller reports as
+// "could not confirm", never as a mismatch: claiming the upgrade failed because the
+// reply was unfamiliar would roll back a perfectly good deploy.
+function parseServedVersion(stdout) {
+  if (!stdout || typeof stdout !== 'string') return null;
+  try {
+    const j = JSON.parse(stdout);
+    for (const k of ['coreVersion', 'core_version', 'version']) {
+      if (typeof j[k] === 'string' && j[k].trim()) return j[k].trim();
+    }
+  } catch { /* not JSON — fall through to the text probe */ }
+  const m = stdout.match(/\b\d+\.\d+\.\d+\b/);
+  return m ? m[0] : null;
+}
+
+/**
+ * The `schemaPending` count from a /version body — migrations the box's code ships that its
+ * own database has not recorded (task 1004153). PURE.
+ *
+ * null means UNKNOWN (no answer, not JSON, or a core older than the field), never 0: a
+ * co-tenant that cannot say must not read as one whose database is current. Bounded the
+ * same way scripts/bongos/deploy-divergence.js bounds it, since the body is a network answer.
+ */
+function parseSchemaPending(stdout) {
+  if (!stdout || typeof stdout !== 'string') return null;
+  let n;
+  try { n = JSON.parse(stdout).schemaPending; } catch { return null; }
+  return Number.isInteger(n) && n >= 0 && n <= 100000 ? n : null;
+}
+
+/**
+ * Is this target a version the instance may actually be moved to? PURE.
+ *
+ * Two separate questions, and conflating them is how a door starts lying:
+ *   • does it EXIST — is it published to the registry at all;
+ *   • is it ALLOWED — does this instance's channel policy admit it (ADR 0136).
+ *
+ * A version that exists but the channel forbids is a refusal the owner can act on
+ * ("your channel is stable and that is a prerelease"); one that does not exist yet is a
+ * refusal they can only wait out. Saying which is which is the difference between a
+ * useful message and "upgrade failed".
+ *
+ * @param {string}   to         the requested version
+ * @param {string[]} available  versions the registry reports
+ * @param {string}   installed  the version running now
+ * @param {string}   channel    this instance's update channel
+ */
+function targetFault(to, available, installed, channel, channelAllows, compare) {
+  if (!Array.isArray(available) || available.length === 0) {
+    return `could not read the published core versions from the registry — nothing was changed; the release may still be publishing (a tag is minted before the publish completes)`;
+  }
+  if (!available.includes(to)) {
+    return `core ${to} is not published yet — nothing was changed. A release tag appears several minutes before the package itself does; try again once it lands.`;
+  }
+  // AN UNREADABLE `installed` IS A FAULT, NOT A FREE PASS (task 1004123).
+  //
+  // This guard used to be written `if (installed && …)`, so a null installed version
+  // skipped the channel check entirely rather than failing it. That is backwards: not
+  // knowing where an instance is today is the one state in which "is this jump
+  // permitted" cannot be answered, and the channel exists precisely to refuse jumps
+  // nobody has looked at. It also hid the defect that motivated this: the control
+  // plane read its installed version from the wrong directory for weeks and got null
+  // every time, and every press sailed past this line.
+  if (!installed) {
+    return `could not read the core version installed at this instance — nothing was changed. An upgrade that cannot say where it is starting from cannot be bounded by the '${channel}' channel.`;
+  }
+  // A TARGET THE BOX HAS ALREADY REACHED IS NOT A CHANNEL QUESTION (task 1004534).
+  // channelAllows answers "may an unattended jump go here" and says no to anything not
+  // strictly newer, so on 2026-10-01 a press for 1.20.73, landed a minute earlier by the
+  // sweep, was reported as "outside this project's 'minor' update channel … across a major
+  // version or onto a prerelease" — false on every count, and it put a "your call" card on
+  // /deploy for a project that was simply up to date. Already there: no fault, so the
+  // caller's already-serving no-op answers it. Already past it: say that, in words that
+  // classify as a downgrade, never as the update rule.
+  if (installed === to) return null;
+  if (compare && compare(to, installed) < 0) {
+    return `core ${to} is older than the core this project already runs (it already runs core ${installed}) — nothing was changed. A move never goes back to an older core on its own.`;
+  }
+  if (channelAllows && !channelAllows(installed, to, channel)) {
+    return `core ${to} is outside this project's '${channel}' update channel — nothing was changed. ${channelNextStep(channel)}`;
+  }
+  return null;
+}
+
+// The step that gets a channel-refused move through, named by the control a person can
+// actually reach (task 1004405). The old line said "change the channel first" while no
+// channel could be changed; the update rule on /deploy (task 1004445) is that control now,
+// and its labels are the words used here. `minor` is the widest rule, so a move it refuses
+// (a major, a prerelease) has no rule to widen to and is named as the by-hand step it is.
+function channelNextStep(channel) {
+  if (channel === 'pinned') {
+    return 'Its update rule on /deploy is "Stay put"; change it to "Fixes only" or "Fixes and new releases", then move again.';
+  }
+  if (channel === 'minor') {
+    return 'No update rule moves a project across a major version or onto a prerelease; that move is made by hand with upgrade.js (docs/recipes/upgrading-the-core.md).';
+  }
+  return 'To take a new release line, set this project\'s update rule on /deploy to "Fixes and new releases" (or press "Allow new releases and move"), then move again.';
+}
+
+// The update channel that bounds a move for THIS instance (task 1004445): the row's own
+// `update_channel`, normalised by update-channel.js so anything unknown or absent (a row
+// read before migration provisioning_029) falls back to DEFAULT_CHANNEL. PURE. One helper
+// for the move and the snapshot, so the button's offer and the runner's refusal cannot
+// disagree about which rule applies.
+function instanceChannel(inst, channelMod) {
+  return channelMod.normalizeChannel(inst && inst.update_channel != null ? inst.update_channel : channelMod.DEFAULT_CHANNEL);
+}
+
+// The versions this project may be moved to: the published ones its FOLLOW admits (task 1004298,
+// ADR 0361). The platform hall (control-plane) takes every candidate; every other project only what
+// has been released, and NOTHING when the registry cannot say which version that is — a project that
+// follows releases is never moved on a guess. A channel module without the follow (an old core
+// resolved at drain time) offers the whole list, which is what every version meant before candidates.
+//
+// The released label is one fact about the registry, the same for every project in a pass, and
+// reading it is a blocking `npm view`. So it is read once per channel module per RELEASE_TAGS_TTL_MS
+// and shared: a pass over N projects costs one extra subprocess, not N. A failed read is not kept.
+const RELEASE_TAGS_TTL_MS = 60 * 1000;
+const releaseTagsMemo = new WeakMap();
+function releaseTagsFor(channelMod, instanceDir, now = Date.now()) {
+  const hit = releaseTagsMemo.get(channelMod);
+  if (hit && now - hit.at < RELEASE_TAGS_TTL_MS) return hit.tags;
+  const tags = channelMod.readReleaseTags({ cwd: instanceDir });
+  if (tags && tags.ok) releaseTagsMemo.set(channelMod, { at: now, tags });
+  return tags;
+}
+
+function offeredVersions(channelMod, inst, instanceDir) {
+  const listed = channelMod.listAvailableVersions({ cwd: instanceDir });
+  if (!listed || !listed.ok || typeof channelMod.followForShape !== 'function') return listed;
+  if (channelMod.followForShape(inst && inst.hosting_shape) === 'candidates') return listed;
+  const tags = releaseTagsFor(channelMod, instanceDir);
+  if (!tags.ok) return { ok: false, error: `could not read which version is released (${tags.error})`, versions: [] };
+  return { ok: true, versions: channelMod.versionsForFollow({ follow: 'released', available: listed.versions, released: tags.released }) };
+}
+
+// A classified refusal (task 1004446): the drain loop stores `failure` beside last_error,
+// so the rest of the self-heal chain acts on a class, not on a sentence. PURE.
+function failed(reason, error, extra = {}) {
+  return { ok: false, error, failure: outcome.classifyFailure(reason), ...extra };
+}
+
+// The builder id a queued move is attributed to, from the actor the web tier recorded.
+// PURE. Two doors reach the move (ADR 0339): an Archon (`api:archon:<id>`) and, since
+// task 1004174, the project's own owner (`api:owner:<id>`). Both carry the id, because
+// it becomes `--requested-by` on the core_upgrades ledger (ADR 0293 D5). Anything else
+// yields '' — the id is interpolated into a shell command, so only digits ever pass.
+function requestedByOf(intent) {
+  const m = intent && typeof intent.requested_by === 'string' && /^api:(?:archon|owner):(\d+)$/.exec(intent.requested_by);
+  return m ? m[1] : '';
+}
+
+/**
+ * Perform a queued core-pin move. Returns { ok } — NEVER throws for an expected refusal.
+ *
+ * The drain loop's catch flips the instance to 'error', which is the wrong outcome for
+ * every refusal here: a request that was never valid, or a version that is not published
+ * yet, says nothing at all about the health of a running instance. So each of those
+ * returns { ok:false, error } and the loop retries or retires the INTENT, leaving the
+ * instance exactly as it found it (ADR 0190 §2, the shape restartInstance established).
+ *
+ * @param {object} inst    the provisioning_instances row
+ * @param {object} deps    the runner's dependency bag
+ * @param {object} intent  the claimed provisioning_intents row (carries target_version)
+ */
+async function coreUpgradeInstance(inst, deps, intent) {
+  const { db, provisioning, exec, log, apply, privileged } = deps;
+  // A standalone's box IS the control plane's box, the same routing restartInstance and
+  // settingsApplyInstance use; a co-tenant shares this checkout.
+  const boxExec = inst.hosting_shape === 'cloud-host' ? (deps.controlExec || exec) : exec;
+  const to = intent && intent.target_version;
+  log(`core-upgrade ${inst.slug} → ${to || '(no target)'} (shape=${inst.hosting_shape}, status=${inst.status})`);
+
+  // ---- 1. is this request allowed at all ------------------------------------------
+  // UPGRADE_SHAPES IS THE WHOLE SHAPE CHECK. The self-row guard is deliberately not
+  // consulted: `intentAllowedForShape('core-upgrade', …)` is true for every shape
+  // there is, because 'core-upgrade' is permanently on CONTROL_PLANE_ACTIONS — so
+  // calling it here would read as a second gate while being unable to refuse
+  // anything. The guard's real job is the OTHER actions on the self row, enforced at
+  // enqueueIntent where every intent passes.
+  if (!UPGRADE_SHAPES.has(inst.hosting_shape)) {
+    return failed('shape_not_automated', `core-upgrade is not automated for a '${inst.hosting_shape}' instance — whoever runs that machine has to upgrade it there`);
+  }
+  if (inst.status !== 'active') {
+    // Not an error: a teardown or a stand-up that raced the request must not burn the
+    // intent's retry budget, and there is no settled pin to move either way.
+    log(`  instance is '${inst.status}' — no running core whose pin can move`);
+    return { ok: true, noop: true };
+  }
+  if (!to) {
+    return failed('no_target', 'core-upgrade intent carries no target_version — refusing to guess which version was meant');
+  }
+  // THE SLUG IS INTERPOLATED INTO A SHELL COMMAND BELOW, so it is re-checked HERE
+  // rather than trusted. `isValidSlug` is enforced on the create route, but the
+  // control-plane self row is inserted by an operator script (enroll-control-plane.js)
+  // and any later write path is one more place that would have to remember. The value
+  // reaching a shell is this function's problem, and a format check at the point of use
+  // is the only one that cannot be bypassed by a path nobody has written yet. The other
+  // two interpolated values are already constrained — the version by the column's own
+  // CHECK and by VERSION_RE, the actor by the digits-only extraction below.
+  if (!isValidSlug(inst.slug)) {
+    return failed('bad_slug', `instance slug '${inst.slug}' is not a valid slug — refusing to build a command from it`);
+  }
+
+  // ---- 2. is the target real, and permitted -----------------------------------------
+  const channelMod = deps.updateChannel || require('./update-channel.js');
+  const selfRoot = deps.selfInstanceRoot || selfInstanceRoot;
+  const instanceDir = inst.hosting_shape === 'cloud-host' ? standaloneRoot(inst) : selfRoot();
+  const readInstalled = deps.readInstalledCoreVersion || require('./upgrade.js').readInstalledCoreVersion;
+  const installed = readInstalled(instanceDir);
+  // THIS PROJECT'S OWN UPDATE RULE (task 1004445, migration provisioning_029). Before it
+  // every instance was bounded by DEFAULT_CHANNEL (patch-only), so a step-up release
+  // could reach no project through the door and the refusal's "change the channel first"
+  // pointed at nothing. A row from before the column (undefined) still normalises to that
+  // default, so an old runner schema reads exactly as it always did.
+  const channel = instanceChannel(inst, channelMod);
+  const listed = offeredVersions(channelMod, inst, instanceDir);
+  const available = (listed && listed.versions) || [];
+  const fault = targetFault(to, available, installed, channel, channelMod.channelAllows, channelMod.compareSemver);
+  if (fault) return failed(outcome.runnerReason(fault), fault);
+  if (installed && installed === to) {
+    // `bongos upgrade` would no-op on this too, but saying so here keeps a re-drained
+    // intent from restarting a healthy service to land exactly where it already is.
+    log(`  already serving core ${to} — nothing to move`);
+    return { ok: true, noop: true };
+  }
+  // ---- 2b. is the database current for the code it runs NOW (task 1004448) -------------
+  // A database behind its own code is already failing every page that touches the missing
+  // tables (the 2026-07-04 shape), and moving the code on top of it tangles a catch-up
+  // migration into a pin move. So it is refused as a known snag, and wedge-remedy.js
+  // migrates that database and tries the move once more. An answer it cannot read proceeds.
+  if (apply) {
+    const probe = boxExec(versionCmd(inst), { allowFail: true });
+    const pending = parseSchemaPending(probe && !probe.softFailed ? (probe.stdout || '') : null);
+    if (pending > 0) return failed('schema_pending', `the project's database is ${pending} migration(s) behind the code it is running now — it has to catch up before the core moves`);
+  }
+
+  // ---- 3. drive the proven path -----------------------------------------------------
+  // Exactly the invocation docs/recipes/core-release-pipeline.md gate 3 documents, plus
+  // the two attribution flags (ADR 0293 D5). --commit-pin is NOT optional here: without
+  // it the pin lives only in the working tree, and the next pull-deploy resets it away —
+  // the silent thirteen-release revert upgrade.js's own header records.
+  const versionUrl = `http://127.0.0.1:${inst.port}/version`;
+  const requestedBy = requestedByOf(intent);
+  // Built by the SAME function the preview uses, so "what you were shown" and "what runs"
+  // cannot drift apart into two hand-maintained flag lists (task 1004121).
+  // The preview probes; the move must probe identically or the two commands diverge on
+  // the one field task 1004121 exists to keep identical. Skipped when !apply, where the
+  // command is printed and never run — a plan should not spend a subprocess per tick.
+  const { runAs, migrateAs } = resolveRunAs(inst, deps, { probe: apply });
+  // A move the hub queued by itself (provision-auto-upgrade.js) says so in the project's
+  // own ledger, so its hall can tell its owner "upgraded automatically" (task 1004731).
+  const automatic = !!intent && intent.requested_by === 'runner:auto-upgrade';
+  const { cmd, runDir } = upgradeInvocation(inst, to, { privileged, requestedBy, automatic, selfRoot, runAs, migrateAs });
+
+  if (!apply) {
+    // A --dry-run runner tick used to only PRINT this command. It now runs the CLI's own
+    // dry run, so `provision.js --dry-run` reports what the upgrade would actually refuse
+    // rather than a command string the reader has to evaluate in their head.
+    log(`  ✓ would run (in ${runDir}): ${cmd}`);
+    const verdict = (deps.preflightUpgrade || preflightUpgrade)(inst, to, deps);
+    for (const line of verdict.lines || []) log(`    ✓ pre-flight: ${line}`);
+    if (!verdict.ok) log(`    ✖ ${verdict.error}`);
+    log(`  ✓ would then read ${versionUrl} on the box and require it to say ${to}`);
+    return { ok: true, served: null, preflight: verdict };
+  }
+  try {
+    log(`  [upgrade] (in ${runDir}) ${cmd}`);
+    boxExec(cmd, { cwd: runDir });
+  } catch (e) {
+    // HOW FAR IT GOT, from upgrade.js's own outcome line (task 1004446) — never assumed.
+    // It used to say "failed and was rolled back" for every exit, including refusals that
+    // fire before the pin moves (nothing changed) and rollbacks that themselves failed
+    // (the project may be down). ok:false, never a throw: the drain loop's catch marks the
+    // INSTANCE broken, which is only ever right for the rollback_failed case, and that one
+    // is reported (and retired, never retried unattended) rather than guessed at here.
+    const text = `${(e && e.stderr) || ''}\n${(e && e.message) || e}`;
+    const o = outcome.parseOutcomeLine(text);
+    const said = (/✖ ([^\n]+)/.exec(text) || [])[1] || String((e && e.message) || e).split('\n')[0];
+    const reason = o ? o.reason : 'unknown';
+    return failed(reason, outcome.describeUpgradeFailure(to, o, said), { stage: o ? o.stage : null, terminal: reason === 'rollback_failed' });
+  }
+
+  // ---- 4. read back what the BOX serves (ADR 0293 D6) --------------------------------
+  // With the same patience the manifest probe uses: the service has just restarted and a
+  // single immediate read races it binding its port. A false "did not take" here is
+  // never cosmetic — it is what the owner is shown after the one act they were told they
+  // could finally perform.
+  const sleep = deps.sleep || ((ms) => new Promise((r) => setTimeout(r, ms)));
+  let served = null;
+  for (let i = 0; i < MANIFEST_VERIFY_TRIES; i++) {
+    const res = boxExec(versionCmd(inst), { allowFail: true });
+    served = parseServedVersion(res && !res.softFailed ? (res.stdout || '') : null);
+    if (served) break;
+    if (i < MANIFEST_VERIFY_TRIES - 1) await sleep(MANIFEST_VERIFY_INTERVAL_MS);
+  }
+  if (served && served !== to) {
+    // upgrade.js's own --version-url gate should have caught and reverted this, so
+    // reaching here means the two disagree. Report what was SEEN rather than a verdict.
+    return failed('served_mismatch', `the box is serving core ${served}, not the requested ${to} — the move did not take`);
+  }
+
+  // ---- 5. the pin reaches the project's own repo, pushed from HERE (task 1004065) ----
+  // Only now: a pin worth saving is one the box is serving. upgrade.js was told
+  // --pin-no-push for this shape, so it committed and stopped; see pushUpgradePin.
+  const repo = hubPushesPin(inst) ? await pushUpgradePin(inst, deps, { instanceDir }) : null;
+  const unsaved = repo && !repo.pushed ? `; GitHub was not updated (${repo.code})` : '';
+  await provisioning.recordEvent(db, {
+    instanceId: inst.id, ownerBuilderId: inst.owner_builder_id, event: 'core-upgraded',
+    detail: (served ? `core pin moved to ${served}` : `core pin moved to ${to}; the box did not answer /version, so this is unconfirmed`) + unsaved,
+    actor: intent.requested_by || 'runner',
+  }).catch(() => {});
+  if (provisioning.clearErrorNote) await provisioning.clearErrorNote(db, inst.id, 'core-upgrade').catch(() => {});
+  log(served ? `  ✓ ${inst.slug} serves core ${served}` : `  ⚠ ${inst.slug} upgraded to ${to}, but did not answer /version — unconfirmed`);
+  // `doneNote`: the drain loop resolves the intent DONE and stores this beside it, which
+  // is how the deploy page learns the upgrade landed but GitHub did not get it.
+  return { ok: true, served, confirmed: served === to, repo, doneNote: repo && !repo.pushed ? repo.note : null };
+}
+
+// ── the pin reaches the project's GitHub repo, pushed from the HUB (task 1004065) ──
+//
+// THE BUG THIS EXISTS FOR. upgrade.js commits the new pin on the box and then pushes it.
+// On a hosted project it runs connected to the PROJECT's database — it has to, for
+// migrations — while the owner's GitHub token lives in the HUB's. Its token lookup found
+// no provisioning table there (ADR 0281), fell back to `git push origin`, and origin is a
+// read-only deploy key (ADR 0176). GitHub refused, the refusal reached only the journal
+// and named the wrong cause, and the intent resolved `done`. The box ran the new core,
+// GitHub still pinned the old one, and the box's local commit stopped its fast-forward
+// pulls from taking the owner's new work: hermeslines-marketing served 1.19.994 against
+// a repo pinning 1.19.881, and pages pushed to that repo 404'd on the live site.
+//
+// WHY THE RUNNER PUSHES rather than handing upgrade.js the token. This process already
+// holds the hub's database, where the token is, and it runs as the account that owns the
+// checkout (the project's own account only READS it — grantInstanceRepoReadCmd). Passing
+// the token into the upgrade would put a credential that can write the repo inside the
+// project's own process. So the move passes --pin-no-push, upgrade.js commits, and the
+// push happens here, in the shape refreshStandaloneCorePin already uses for this repo.
+//
+// A FAILED PUSH IS NOT A FAILED UPGRADE. The site runs the new core, and rolling a good
+// deploy back would be worse than the split. The move still resolves `done`, carrying a
+// note the deploy page reads: `repo_not_updated:<code>: <sentence>`, parsed by
+// parseRepoNote in modules/provisioning/core-upgrade.js (re-stated there because a
+// module may not require a script; tests/core_upgrade_runner.mjs round-trips the two).
+//
+// NOTHING IS FORCED, AND THE LIVE BOX IS NEVER REBASED. When GitHub has commits the box
+// does not, moving the box onto them would change what it runs with no restart and no
+// health check. Since task 1004733 the pin commit ALONE is replayed onto GitHub's history in
+// a scratch worktree and pushed as an ordinary fast-forward (provision-pin-replay.js), so
+// GitHub names the new core and the box stays exactly what the move proved. A conflict in
+// the pin files is reported ("conflict"), never resolved here. The box still does not pick
+// up the owner's other commits — that would be a deploy, not a pin push (ADR 0350 amendment).
+//
+// WHAT MAKES IT DURABLE (task 1004291, ADR 0350). The owner's token is kept for an hour
+// after they sign in with repository access (GITHUB_TOKEN_TTL_MS in src/bongos/db.js — one
+// wizard sitting), so on its own it cannot carry an upgrade days later. The hub therefore
+// holds a per-project WRITE deploy key (provision-pin-key.js) and tries it FIRST: write key
+// → owner token (when fresh) → the box's own remote, which can only read a private repo.
+// A fresh token also mints the key when the project has none, so reconnecting GitHub once
+// is enough for every later upgrade. The note says which of these is missing, in words.
+
+const REPO_NOTE_PREFIX = 'repo_not_updated';
+const REPO_NOTES = Object.freeze({
+  no_repo: 'This project has no GitHub repository on record, so the new version was saved on the server only.',
+  no_branch: 'The server\'s copy of the project is not on a branch, so there was nothing to save to GitHub.',
+  no_access: 'The hub had no permission to write to the project\'s GitHub repository: it holds no write key for it yet, the owner\'s GitHub access is only kept for an hour after they connect GitHub, and the server\'s own key can only read the repository. Reconnecting GitHub with repository access sets up lasting access.',
+  needs_account: 'The hub could not set up lasting write access to the project\'s GitHub repository: this project needs its own account on the server first. Once it has one, reconnecting GitHub with repository access sets up lasting access.',
+  refused: 'GitHub turned down the owner\'s GitHub access — it may have expired, or lack permission to write to the repository.',
+  behind: 'GitHub has changes the server does not have, so the new version could not be added on top of them. An operator needs to bring the two back in line; the site itself is running normally.',
+  conflict: 'GitHub has its own changes to the same version files (package.json or package-lock.json), so the new version was not added on top of them and nothing there was overwritten. An operator needs to choose between the two; the site itself is running normally.',
+  failed: 'The hub could not save the new version to GitHub.',
+});
+// A branch name that is safe to interpolate. Narrower than git's own ref grammar on
+// purpose: a name outside it is refused, never quoted.
+const BRANCH_RE = /^[A-Za-z0-9][A-Za-z0-9._/-]{0,99}$/;
+
+/** The note a done-but-unsaved move carries, bounded for the DOM it reaches. PURE. */
+function repoNote(code, extra = '') {
+  const known = Object.prototype.hasOwnProperty.call(REPO_NOTES, code) ? code : 'failed';
+  return `${REPO_NOTE_PREFIX}:${known}: ${REPO_NOTES[known]}${extra ? ` ${extra}` : ''}`.slice(0, 600);
+}
+
+// Which shapes get their pin pushed from here: a hosted project, whose repo is the
+// owner's own. The platform's own row pushes to a repo its key CAN write, and a
+// co-tenant shares the platform's checkout, so neither changes.
+function hubPushesPin(inst) { return !!inst && inst.hosting_shape === 'cloud-host'; }
+
+// Why a push did not happen, from what git said. `behind` is checked first: a rejected
+// update means the credential WORKED and the history is what stopped it.
+function pushFault(r) {
+  const said = `${(r && r.stdout) || ''}\n${(r && r.stderr) || ''}`;
+  if (/\[rejected\]|non-fast-forward|fetch first|Updates were rejected/i.test(said)) return 'behind';
+  if (/\b403\b|Authentication failed|Invalid username or password|Write access to repository not granted|Permission to \S+ denied|marked as read only|Permission denied \(publickey\)|could not read Username/i.test(said)) return 'refused';
+  return 'failed';
+}
+
+// GitHub's own line beats git's closing `fatal:`, which names no reason (upgrade.js
+// gitSaid has the same preference).
+function gitLine(r) {
+  const lines = String((r && r.stderr) || '').split('\n').map((l) => l.trim()).filter(Boolean);
+  const server = lines.filter((l) => /^(ERROR:|remote:|!)/i.test(l));
+  const said = server.length ? server[server.length - 1] : lines[lines.length - 1];
+  return said ? `(git: ${said.slice(0, 200)})` : '';
+}
+
+/**
+ * Push a confirmed upgrade's committed pin to the project's repo. NEVER THROWS: every
+ * miss returns `{ pushed:false, code, note }`, because the drain loop turns a throw into
+ * a broken instance and this instance is running fine.
+ *
+ * The token rides a chmod-600 credential file that is removed whatever the push did —
+ * never argv (ps, the journal), never a log line. The helper list is reset first
+ * (helpers are cumulative, so a global one would answer with some other account) and
+ * hooks are off, so nothing in the project's checkout runs as this account.
+ */
+async function pushUpgradePin(inst, deps, { instanceDir }) {
+  const log = deps.log || (() => {});
+  // The CONTROL PLANE's executor and writer: the checkout, the hub db and the token are
+  // all here, never on a remote co-hosting box (refreshStandaloneCorePin's routing).
+  const exec = deps.controlExec || deps.exec;
+  const writeFile = deps.controlWriteFile || deps.writeFile || ((p, body, o) => fs.writeFileSync(p, body, o));
+  const failedRun = (r) => !r || r.softFailed === true || r.ok === false;
+  const miss = (code, extra = '') => {
+    log(`  ⚠ ${inst.slug}: GitHub was not updated — ${REPO_NOTES[code] || REPO_NOTES.failed}${extra ? ` ${extra}` : ''}`);
+    return { pushed: false, code, note: repoNote(code, extra) };
+  };
+  const repo = parseTargetRef(inst.target_ref);
+  if (!repo) return miss('no_repo');
+  // Exported and callable on its own, and the slug reaches shell strings below (the write
+  // key's path), so it is checked here rather than trusting coreUpgradeInstance's own guard.
+  if (!isValidSlug(inst.slug)) return miss('failed', '(the project name is not a valid slug)');
+  const head = exec(`git -C ${instanceDir} symbolic-ref --short HEAD`, { allowFail: true });
+  const branch = failedRun(head) ? '' : String(head.stdout || '').trim();
+  if (!BRANCH_RE.test(branch)) return miss('no_branch');
+  const at = `${repo.owner}/${repo.name}@${branch}`;
+  const hardened = `git -C ${instanceDir} -c core.hooksPath=/dev/null`;
+  const env = { GIT_TERMINAL_PROMPT: '0' };   // a refused credential fails; it never waits at a prompt
+  // GitHub is ahead: replay the pin commit alone on top, over the credential that was just
+  // refused as behind, and push again — never forced (task 1004733, provision-pin-replay.js).
+  const replayed = (via, git, remote) => {
+    const rr = replayPinCommit({ exec, instanceDir, branch, git, remote, env, slug: inst.slug, tmpDir: deps.tmpDir, pushFault, gitLine, log });
+    if (!rr.ok) return miss(rr.code, rr.extra);
+    if (rr.already) log(`  ✓ ${at} already names this version — nothing to add`);
+    return { pushed: true, via, branch, ...(rr.replayed ? { replayed: true } : { already: true }) };
+  };
+
+  const token = await resolveOwnerGithubToken(inst, deps, { tag: '[core-upgrade]' });
+
+  // 1. The hub's own WRITE key (ADR 0350). A fresh token mints it when the project has
+  // none yet — the reconnect the deploy page asks for — and the mint is best effort: a
+  // miss falls through to the token, then to the box's remote, exactly as before.
+  const fsImpl = deps.fsImpl || fs;
+  const keyPath = pinKeyPath(inst, deps);
+  if (token && !fsImpl.existsSync(keyPath)) await ensurePinWriteKey(inst, deps, { token });
+  if (fsImpl.existsSync(keyPath)) {
+    const r = exec(`${hardened} -c ${pinSshCommand(keyPath)} push git@github.com:${repo.owner}/${repo.name}.git HEAD:${branch}`,
+      { allowFail: true, env });
+    if (!failedRun(r)) {
+      log(`  ✓ pushed the pin to ${at} from the hub, over its write deploy key`);
+      return { pushed: true, via: 'write-key', branch };
+    }
+    const fault = pushFault(r);
+    // The key WORKED and the history stopped it: no other credential changes that, so the
+    // pin is replayed on top of GitHub's history over this same key (task 1004733).
+    if (fault === 'behind') return replayed('write-key', `${hardened} -c ${pinSshCommand(keyPath)}`, `git@github.com:${repo.owner}/${repo.name}.git`);
+    log(`  • ${inst.slug}: the write key did not push (${fault}) — trying the next credential`);
+  }
+
+  if (token) {
+    const credFile = path.join(os.tmpdir(), `bongos-gitcred-upgrade-${inst.slug}-${process.pid}-${crypto.randomBytes(6).toString('hex')}`);
+    let r = null;
+    try {
+      writeFile(credFile, `https://x-access-token:${token}@github.com\n`, { mode: 0o600 });
+      const viaToken = `${hardened} -c credential.helper= -c credential.helper='store --file=${credFile}'`;
+      const tokenRemote = `https://x-access-token@github.com/${repo.owner}/${repo.name}.git`;
+      r = exec(`${viaToken} push ${tokenRemote} HEAD:${branch}`, { allowFail: true, env });
+      // Behind: replayed while the credential file still exists (the finally removes it).
+      if (failedRun(r) && pushFault(r) === 'behind') return replayed('owner-token', viaToken, tokenRemote);
+    } catch (e) {
+      return miss('failed', `(could not stage the owner's credential: ${String((e && e.message) || e).slice(0, 160)})`);
+    } finally {
+      exec(`rm -f ${credFile}`, { allowFail: true });
+    }
+    if (!failedRun(r)) {
+      log(`  ✓ pushed the pin to ${at} from the hub, over the owner's GitHub token`);
+      return { pushed: true, via: 'owner-token', branch };
+    }
+    return miss(pushFault(r), gitLine(r));
+  }
+
+  // No token stored — which is the ordinary case (see THE LIMIT above). The box's own
+  // remote is still tried: a checkout wired with a key that can write is not wrong to use.
+  const r = exec(`${hardened} push origin HEAD:${branch}`, { allowFail: true, env });
+  if (!failedRun(r)) {
+    log(`  ✓ pushed the pin to ${at} from the hub, over the checkout's own remote`);
+    return { pushed: true, via: 'origin', branch };
+  }
+  const fault = pushFault(r);
+  if (fault === 'behind') return replayed('origin', hardened, 'origin');
+  // A refusal here is not the owner's access being turned down — there was none to turn
+  // down. It is the read-only key, which is the no-access case.
+  // ...unless the project has no account of its own, in which case no key CAN be minted
+  // for it (ADR 0350) and reconnecting GitHub alone will not help: say that instead.
+  if (fault === 'refused') return miss(hasOwnAccount(inst, exec) ? 'no_access' : 'needs_account', gitLine(r));
+  return miss(fault, gitLine(r));
+}
+
+// The pre-flight lines a dry run emits, and the refusal when it has one. upgrade.js writes
+// its pre-flight to stdout as `  ✓ pre-flight: …` and returns a `✖`-prefixed multi-line
+// error on a refusal, so BOTH streams matter: a pass is only legible from stdout, and a
+// refusal only from the failure text. PURE given its input.
+function parsePreflight(out) {
+  const text = String(out || '');
+  const lines = text.split('\n').map((l) => l.trim());
+  const checks = lines.filter((l) => /^[✓✖]\s*pre-flight:/.test(l)).map((l) => l.replace(/^[✓✖]\s*/, ''));
+  // A refusal is whatever upgrade.js said after the ✖ — reported verbatim rather than
+  // re-worded, because its wording already names the fix ("Set PGDATABASE, or …").
+  const failAt = text.indexOf('✖');
+  const error = failAt >= 0 ? text.slice(failAt + 1).trim() : null;
+  return { checks, error };
+}
+
+/**
+ * Which unix accounts does this instance's upgrade need? Asks the box (task 1004128).
+ *
+ * TWO ANSWERS, NOT ONE (task 1004512). Task 1004128 asked a single question — "which
+ * account does this instance run as?" — and ran the WHOLE move as it. That is right for
+ * the database and wrong for the files, and the two have different owners on every
+ * project provisioned since task 1003369:
+ *
+ *   • `runAs` — the account that owns the CHECKOUT, which is the app user on every
+ *     hosting shape (provision-repo.js scaffolds /srv/<base>/<slug> as the runner, and
+ *     grantInstanceRepoReadCmd gives the project's own account group READ, nothing more).
+ *     Everything the move writes — the pin, node_modules, .claude/, the pin commit — and
+ *     the `git status` it starts with, belong to this account. Running them as the
+ *     project's account is what made every such move die on `fatal: detected dubious
+ *     ownership in repository` and report a dirty tree that did not exist.
+ *   • `migrateAs` — the project's OWN account, which is also its Postgres login role
+ *     (provision-repo.js instanceDbRole); its database is owned by that role with CONNECT
+ *     revoked from PUBLIC, so this is the only identity that can migrate it. That is where
+ *     the isolation of task 1003369 actually lives, and it is kept.
+ *
+ * The fleet is heterogeneous: instances provisioned before task 1003369 run as the
+ * shared app user, instances provisioned after run as their own `bongos-<slug>`. Neither
+ * is guessable from the row, and getting it wrong is not a soft failure — `sudo` refuses
+ * an unknown user and the upgrade dies before pre-flight, which is exactly how the
+ * tenant leg has been broken.
+ *
+ * Falling back is NOT a privilege escalation: an instance with no account of its own is
+ * already running as the app user (its unit says so), so this migrates as precisely the
+ * account that owns the database it is about to touch. It IS reported, because a missing
+ * per-instance account is a provisioning gap someone should close, and a silent fallback
+ * would hide it forever. On that shape the two answers coincide and the move is byte-for-
+ * byte what it was before this task.
+ */
+function resolveRunAs(inst, deps, { probe: allowProbe = true } = {}) {
+  const owner = appUser(); // owns the checkout on every shape — see above
+  if (inst.hosting_shape === 'control-plane') return { runAs: owner, migrateAs: owner, fellBack: false };
+  // Re-checked HERE rather than left to instanceUser()'s own throw, for the reason
+  // upgradeInvocation states about itself: a call site whose safety depends on a
+  // helper's internals is one refactor away from a shell-interpolation path.
+  if (!isValidSlug(inst && inst.slug)) {
+    throw new Error(`resolveRunAs: refusing to probe an account for invalid slug ${JSON.stringify(inst && inst.slug)}`);
+  }
+  const wanted = instanceUser(inst);
+  if (!allowProbe) return { runAs: owner, migrateAs: wanted, fellBack: false };
+  // THE SAME EXECUTOR THE UPGRADE ITSELF WILL USE. Every sibling here picks
+  // `standalone ? (controlExec || exec) : exec`, because under PROVISION_REMOTE_HOST
+  // (ADR 0130) `exec` is the REMOTE ssh executor while `controlExec` stays local. A
+  // probe on `exec` would then ask the co-hosting box whether an account exists while
+  // the command it is deciding for runs on the control plane — a wrong answer that
+  // looks like a right one.
+  const boxExec = inst.hosting_shape === 'cloud-host' ? (deps.controlExec || deps.exec) : deps.exec;
+  const r = boxExec(`id -u ${wanted}`, { allowFail: true });
+  const failed = !r || r.softFailed === true || r.ok === false;
+  if (!failed) return { runAs: owner, migrateAs: wanted, fellBack: false };
+  // ONLY A DEFINITIVE "no such user" JUSTIFIES THE FALLBACK. A probe that could not
+  // answer — an exec timeout, an NSS hiccup — must not silently narrow the per-instance
+  // isolation of task 1003369 for this operation. So an inconclusive probe keeps the
+  // intended account: if it really is missing the upgrade fails loudly, and since this
+  // task a failure now carries the reason instead of "the dry run did not complete".
+  const said = `${(r && r.stdout) || ''}\n${(r && r.stderr) || ''}`;
+  if (!/no such user|unknown user/i.test(said)) {
+    return { runAs: owner, migrateAs: wanted, fellBack: false, probeInconclusive: true };
+  }
+  return { runAs: owner, migrateAs: owner, fellBack: true, wanted };
+}
+
+// ONE BUILDER FOR BOTH LEGS (task 1004121).
+//
+// The preview and the move must be the same command or the preview is worthless — and
+// two hand-copied builders are exactly how they stop being the same. The mode decides
+// only the TAIL: a dry run stops after pre-flight, so the flags that describe what to do
+// AFTER the pin moves (restart, health, version, pin-commit, attribution) would be
+// inert on it. Everything that decides whether the upgrade is ALLOWED — the account, the
+// database, the target, the instance — is shared, which is the half a preview is about.
+//
+// The slug rides into a shell command through instanceUser()/dbName(), so it is checked
+// HERE rather than left to instanceUser()'s own throw: that throw is correct today, but
+// a call site whose safety depends on a helper's internals is one refactor away from a
+// shell-injection path (and the apply leg's own guard sits at the route, not here).
+function upgradeInvocation(inst, to, { privileged = false, dryRun = false, requestedBy = '', automatic = false, selfRoot = selfInstanceRoot, runAs = null, migrateAs = null } = {}) {
+  if (!isValidSlug(inst && inst.slug)) {
+    throw new Error(`upgradeInvocation: refusing to build a shell command from an invalid slug ${JSON.stringify(inst && inst.slug)}`);
+  }
+  // WHERE THE CLI LIVES AND WHERE THE TARGET LIVES ARE DIFFERENT PLACES, TWICE OVER.
+  //
+  // `upgrade.js` ships in the CORE, and a provisioned instance root holds no scripts/
+  // directory at all — so the CLI runs from REPO_ROOT and NAMES its target (task
+  // 1004053, which fixed every standalone tenant).
+  //
+  // The second half is that REPO_ROOT is not the instance either (task 1004123). It
+  // names the copy of the core that is EXECUTING, which coincides with the instance
+  // only while the core is a checkout run in place. Once the control plane's runner
+  // moved into its vendored core the two diverged silently — REPO_ROOT still resolved,
+  // just to node_modules/@bongos/core — and the door aimed at the package instead of
+  // the project. So ask, rather than assume; in the legacy shape the answer is
+  // REPO_ROOT and the command below is byte-identical to what it always was.
+  const instanceDir = inst.hosting_shape === 'cloud-host' ? standaloneRoot(inst) : selfRoot();
+  // WHICH ACCOUNT, AND WHY IT IS RESOLVED RATHER THAN DERIVED (task 1004128).
+  //
+  // Task 1004119 wrote this as `control-plane ? APP_USER : instanceUser(inst)` and was
+  // half right. The control-plane leg is correct and verified live. The tenant leg
+  // assumed the per-instance account model of task 1003369 is provisioned everywhere —
+  // and on the live box NO `bongos-*` account exists, no such Postgres role exists, and
+  // every instance unit still names the shared app user. So the fix for "Postgres has no role
+  // root" swapped one absent account for another, and every tenant upgrade died with
+  // `sudo: unknown user bongos-test4`.
+  //
+  // The account model is a real security control, not an accident — a distinct uid per
+  // instance is what stops a co-tenant reading a sibling's /proc/<pid>/environ. So the
+  // answer is NOT to route everyone back to the shared account. It is to run as the
+  // account the instance ACTUALLY runs as, which a caller establishes by probing
+  // (resolveRunAs below) instead of either side assuming. Passing it in keeps this
+  // builder pure; the default preserves the old derivation for callers that have not
+  // probed.
+  //
+  // AND WHICH ACCOUNT DOES WHAT (task 1004512). `account` runs the move; `dbAccount` runs
+  // its three database steps. They differ on exactly the projects that have an account of
+  // their own, because the checkout is the app user's and the database is theirs —
+  // resolveRunAs above argues the split. `--migrate-as` is emitted ONLY when they differ,
+  // so a control plane and a pre-1003369 project produce the command they always did.
+  const account = runAs || appUser();
+  const dbAccount = migrateAs || (inst.hosting_shape === 'control-plane' ? appUser() : instanceUser(inst));
+  const sudoP = privileged ? `sudo -u ${account} env PGDATABASE=${dbName(inst)} ` : '';
+  const cli = path.posix.join('scripts', 'bongos', 'upgrade.js');
+  const shared = [`${sudoP}node ${cli}`, '--registry', `--to ${to}`,
+    instanceDir === REPO_ROOT ? '' : `--instance ${instanceDir}`,  // REPO_ROOT is where the CLI runs (runDir below)
+    // SHARED, not in the tail: the dry run's own database pre-check connects as this
+    // account too, so a preview that omitted it would test a different identity than the
+    // move — the exact drift one builder for both legs exists to prevent.
+    dbAccount === account ? '' : `--migrate-as ${dbAccount}`];
+  const tail = dryRun
+    ? ['--dry-run']
+    : [`--service ${inst.slug}`,
+      `--health-url http://127.0.0.1:${inst.port}/healthz`,
+      `--version-url http://127.0.0.1:${inst.port}/version`,
+      '--commit-pin',
+      // A hosted project's pin is pushed by the RUNNER, which can reach the owner's
+      // token; the upgrade process cannot (task 1004065, pushUpgradePin).
+      hubPushesPin(inst) ? '--pin-no-push' : '',
+      requestedBy ? `--requested-by ${requestedBy}` : '',
+      requestedBy ? '--upgrade-source owner-control' : '',
+      // The ledger's existing mark for an unattended move (the self-host sweep writes
+      // `auto-subscription (<rule>)`); one shell-safe word here (task 1004731).
+      automatic && !requestedBy ? '--note auto-subscription' : ''];
+  return { cmd: [...shared, ...tail].filter(Boolean).join(' '), runDir: REPO_ROOT, instanceDir, runAs: account, migrateAs: dbAccount };
+}
+
+// Ask the upgrade CLI what it WOULD do, changing nothing. `allowFail` because a refusal
+// is the interesting answer, not an exception: pre-flight refusing is what this is for.
+function preflightUpgrade(inst, to, deps) {
+  const { exec, privileged } = deps;
+  const boxExec = inst.hosting_shape === 'cloud-host' ? (deps.controlExec || exec) : exec;
+  // A distinct local name: `deps.selfInstanceRoot || selfInstanceRoot` with a const of
+  // the same name would TDZ-shadow the import and crash every uninjected call.
+  const selfRoot = deps.selfInstanceRoot || selfInstanceRoot;
+  const { runAs, migrateAs, fellBack, wanted } = resolveRunAs(inst, deps);
+  const { cmd, runDir } = upgradeInvocation(inst, to, { privileged, dryRun: true, selfRoot, runAs, migrateAs });
+  const r = boxExec(cmd, { cwd: runDir, allowFail: true });
+  const out = `${(r && r.stdout) || ''}\n${(r && r.stderr) || ''}`;
+  const { checks, error } = parsePreflight(out);
+  // ok is the RUN's verdict, not the parser's: a command that exited non-zero without a
+  // recognisable ✖ is still a refusal, and reporting it as a pass is the failure mode
+  // this whole task exists to remove.
+  const ok = !!(r && !r.softFailed && r.ok !== false) && !error;
+  // WHEN THERE IS NO ✖, SAY WHAT THE COMMAND SAID (task 1004128). The generic sentence
+  // is the LAST resort, not the first: the first live pre-flight reported "the dry run
+  // did not complete" while `sudo: unknown user bongos-test4` sat in the journal, and
+  // the second did the same over `✖ already on 1.19.927`. Both were in hand. The trim
+  // is because this reaches a DOM.
+  const raw = out.trim().split('\n').filter(Boolean).slice(-4).join('\n').slice(0, 600);
+  const why = error || (raw || null);
+  const note = fellBack
+    ? `migrated as ${migrateAs} — this instance has no ${wanted} account yet (it predates the per-instance account model)`
+    : null;
+  return {
+    // Both accounts, because since task 1004512 there are two and "which user did this run
+    // as" has no single answer: `ran_as` writes the files, `migrated_as` touches the database.
+    target: to, ok, lines: checks, ran_as: runAs, migrated_as: migrateAs, account_note: note,
+    error: ok ? null : (why || 'the dry run did not complete, and said nothing this could report'),
+  };
+}
+
+// ── WHAT THE MOVE WOULD CARRY (task 1004173) ────────────────────────────────
+//
+// The owner, 2026-09-22: "I don't actually see what would change when I hit the button."
+// The pre-flight above answers a different question — whether the machinery will let the
+// move happen — and answering it was being presented as a preview. ADR 0293 D4 promised
+// both halves in so many words: "the control shows what a bump would move — live version
+// → target, AND WHAT SITS BETWEEN THEM." Only the first half was built, so the owner was
+// asked to restart a live site to take ninety-one releases of work the page named none of.
+//
+// WHERE THE ANSWER ALREADY LIVES. Every ship writes a merge commit whose summary is
+// already prose for exactly this reader — the owner, not a developer — because ship.js
+// requires one and posts it to Discord. So this reads the merge summaries between the two
+// releases rather than inventing a changelog nobody would maintain.
+//
+// IT READS THE NOTES THE CORE PACKAGE CARRIES, and a core checkout only as a fallback
+// (task 1004218). "What changed in the core between 1.19.881 and 1.19.972" is a question
+// about the CORE repository, and no tenant box will ever hold its history — every project
+// is a checkout of its OWN repo with the core vendored in from npm. So the packer writes
+// the summaries down once, from the full history CI packs with (release-notes.json, built
+// by scripts/bongos/release-notes.js), and any machine that can download a core version can
+// read what it carries. The checkout path below is kept for a machine that has one.
+const CHANGE_CAP = 20;
+// One registry download of a core tarball. The preview's owner is waiting on the dry run
+// beside it anyway, but a hung registry must not hold the runner forever.
+const NOTES_FETCH_TIMEOUT_MS = 60000;
+
+function runFault(r) {
+  if (!r) return 'no answer';
+  if (r.error) return String(r.error.message || r.error).slice(0, 160);
+  const tail = String(r.stderr || '').trim().split('\n').filter(Boolean).slice(-1)[0] || '';
+  return (tail || `exit ${r.status}`).slice(0, 160);
+}
+
+/**
+ * The release notes the core THIS runner executes from carries, if any. Free to read, and
+ * enough on its own whenever the target is at or below this core's own version — a tenant
+ * catching up to where the platform already is.
+ */
+function readInstalledNotes(deps = {}) {
+  const fsImpl = deps.fs || fs;
+  let raw;
+  try { raw = fsImpl.readFileSync(path.join(REPO_ROOT, RELEASE_NOTES_FILE), 'utf8'); }
+  catch { return { missing: 'this core carries no release notes' }; }
+  try { return { notes: JSON.parse(raw) }; }
+  catch { return { error: "this core's release notes are not valid JSON" }; }
+}
+
+/**
+ * The release notes inside ONE published core package — the TARGET's own (task 1004218).
+ *
+ * WHY THE TARGET'S. The installed core's notes stop at its own version, which is zero
+ * coverage for the move looked at most — the platform's own, onto a release newer than
+ * itself. The target's package is the one guaranteed to reach the target, because that is
+ * where the target was packed. `npm pack` downloads it (the core is public, so no
+ * credential — the same registry listAvailableVersions reads) and `tar -xzOf` reads the one
+ * file out of it rather than unpacking the rest.
+ *
+ * Returns `{ notes }`, `{ missing }` when the package carries none (every release published
+ * before the notes were packaged), or `{ error }`. The download never outlives the call.
+ */
+function readPackagedNotes(version, deps = {}) {
+  if (!CORE_VERSION_RE.test(String(version || ''))) return { error: 'could not work out which package to read' };
+  const run = deps.spawn || spawnSync;
+  const fsImpl = deps.fs || fs;
+  let dir = null;
+  try {
+    dir = fsImpl.mkdtempSync(path.join(deps.tmpDir || os.tmpdir(), 'bongos-release-notes-'));
+    const packed = run('npm', ['pack', `@bongos/core@${version}`, '--pack-destination', dir, '--silent'],
+      { cwd: dir, env: deps.env || process.env, encoding: 'utf8', timeout: NOTES_FETCH_TIMEOUT_MS });
+    if (!packed || packed.error || packed.status !== 0) {
+      return { error: `could not download the core ${version} package to read its release notes (${runFault(packed)})` };
+    }
+    // Whatever npm named it: a scoped package's tarball is <scope>-<name>-<version>.tgz today.
+    const tgz = fsImpl.readdirSync(dir).find((f) => f.endsWith('.tgz'));
+    if (!tgz) return { error: `npm reported the core ${version} package downloaded, but left no tarball` };
+    const x = run('tar', ['-xzOf', path.join(dir, tgz), `package/${RELEASE_NOTES_FILE}`],
+      { encoding: 'utf8', maxBuffer: 4 * RELEASE_NOTES_MAX_BYTES, timeout: NOTES_FETCH_TIMEOUT_MS });
+    if (!x || x.error || x.status !== 0) {
+      if (/not found in archive/i.test(String((x && x.stderr) || ''))) {
+        return { missing: `the core ${version} package carries no release notes` };
+      }
+      return { error: `could not read the release notes out of the core ${version} package (${runFault(x)})` };
+    }
+    try { return { notes: JSON.parse(x.stdout) }; }
+    catch { return { error: `the release notes in the core ${version} package are not valid JSON` }; }
+  } catch (e) {
+    return { error: `could not fetch the core ${version} package (${String((e && e.message) || e).slice(0, 160)})` };
+  } finally {
+    if (dir) { try { fsImpl.rmSync(dir, { recursive: true, force: true }); } catch { /* best-effort */ } }
+  }
+}
+
+/**
+ * The range, answered from release notes alone: `{ entries }`, or `{ why }` they cannot.
+ * The installed core first, when its notes already reach the target; the target's own
+ * package otherwise. No git, no SHAs, no checkout.
+ */
+function changesFromNotes(fromVersion, toVersion, deps) {
+  const installed = (deps.readInstalledNotes || readInstalledNotes)(deps);
+  if (installed.notes) {
+    const r = notesBetween(installed.notes, fromVersion, toVersion);
+    if (r.entries) return r;
+  }
+  const target = (deps.readPackagedNotes || readPackagedNotes)(toVersion, deps);
+  if (!target.notes) return { why: target.missing || target.error || `the core ${toVersion} package's release notes could not be read` };
+  const r = notesBetween(target.notes, fromVersion, toVersion);
+  return r.entries ? r : { why: `the release notes in the core ${toVersion} package ${r.uncovered}` };
+}
+
+/**
+ * Classify a git command that did not succeed. PURE.
+ *
+ * "COULD NOT LOOK" AND "LOOKED AND DID NOT FIND" ARE DIFFERENT ANSWERS (task 1004212).
+ * The first cut collapsed both into "could not find those releases in the core history on
+ * this machine", and that sentence sent the investigation at the history for an hour when
+ * the history was never the problem — the directory was. A message that names the wrong
+ * cause is worse than a vague one, because it is followed.
+ */
+function gitFault(r, dir) {
+  const said = `${(r && r.stdout) || ''}\n${(r && r.stderr) || ''}`;
+  if (/dubious ownership|safe\.directory/i.test(said)) {
+    return `the core checkout at ${dir} belongs to another account, so this could not read it`;
+  }
+  if (/not a git repository/i.test(said)) return `${dir} is not a git checkout`;
+  const tail = said.trim().split('\n').filter(Boolean).slice(-1)[0] || '';
+  return `could not read ${dir}${tail ? ` — ${tail.slice(0, 160)}` : ''}`;
+}
+
+/**
+ * Where the CORE's own git history lives on this machine, if anywhere (task 1004212).
+ *
+ * WHAT THE FIRST CUT GOT WRONG, and it was not a small thing. It read `selfInstanceRoot()`
+ * on the assumption that the control plane's instance root is a checkout of the core. On
+ * the live box it is not: `/version` there reports a commit that does not exist in the core
+ * repository at all, because the instance root is a checkout of the INSTANCE's own repo
+ * with the core vendored in at node_modules (the task 1004123 shape). So every lookup
+ * searched the wrong repository's history.
+ *
+ * THAT IS A CORRECTNESS BUG, NOT ONLY A MISSING FEATURE. On a box whose instance repo
+ * happened to carry `Merge <branch>: <summary>` commits — which any repo built this way
+ * does — the page would have shown ANOTHER project's changelog as the core's, with no
+ * error at all. Which is why a candidate is not trusted for being a git repo: it has to
+ * prove it carries core RELEASE history, and `looksLikeCore` is that proof.
+ *
+ * NO PATH IS HARDCODED. `audit-authorship.sh` forbids naming the founder's directories,
+ * and a guessed path is how this got here. The candidates are: an explicit operator
+ * setting, the directory the CLI is executing from, and the instance root — each checked,
+ * none assumed.
+ */
+function coreCheckoutCandidates(deps) {
+  const selfRoot = deps.selfInstanceRoot || selfInstanceRoot;
+  const env = deps.env || process.env;
+  const out = [];
+  const add = (dir, why) => { if (dir && !out.some((c) => c.dir === dir)) out.push({ dir, why }); };
+  add(String(env.PROVISION_CORE_CHECKOUT || '').trim() || null, 'PROVISION_CORE_CHECKOUT');
+  add(REPO_ROOT, 'the directory this core is running from');
+  add(selfRoot(), "this project's own root");
+  return out;
+}
+
+/**
+ * Read the core's release history, or say precisely why it cannot be. Returns
+ * `{ dir }` on success, or `{ error }` — never a directory it has not verified.
+ */
+function findCoreCheckout(deps, git) {
+  const tried = [];
+  for (const { dir, why } of coreCheckoutCandidates(deps)) {
+    const probe = git(dir, 'rev-parse --is-shallow-repository');
+    if (!probe.ok) { tried.push(`${why}: ${gitFault(probe, dir)}`); continue; }
+    if (probe.stdout.trim() === 'true') {
+      // A depth-limited clone resolves HEAD perfectly well — which is why /version reports
+      // `resolvedVia: git` on a box that cannot answer this — and carries no old releases.
+      tried.push(`${why}: ${dir} is a shallow clone, so it carries no release history`);
+      continue;
+    }
+    // THE PROOF THAT THIS IS THE CORE. Any repository answers `rev-parse`; only the core's
+    // carries release commits, and accepting a repo without asking is how the wrong
+    // project's changelog would have been shown as this one's.
+    const marker = git(dir, "log --format=%H --max-count=1 -E --grep='^release: core [0-9]+\\.[0-9]+\\.[0-9]+ '");
+    if (!marker.ok) { tried.push(`${why}: ${gitFault(marker, dir)}`); continue; }
+    if (!marker.stdout.trim()) { tried.push(`${why}: ${dir} is a git checkout, but not of the core`); continue; }
+    return { dir };
+  }
+  return { error: `no core checkout is reachable on this machine (${tried.join('; ')}). Set PROVISION_CORE_CHECKOUT to one.` };
+}
+
+/**
+ * The range, answered from a core CHECKOUT's git history: `{ entries }` or `{ error }`.
+ * The fallback for a machine that has one, since the package's notes only exist for
+ * releases published after they were introduced (task 1004218).
+ */
+function changesFromCheckout(fromVersion, toVersion, deps) {
+  // The CONTROL PLANE's executor, never the previewed instance's: under
+  // PROVISION_REMOTE_HOST `exec` is the remote box, while the core history — and the whole
+  // question of what changed in the CORE — belongs to the machine that builds it.
+  const exec = deps.controlExec || deps.exec;
+  // AS THE ACCOUNT THAT OWNS THE CHECKOUT, for the same reason the dry run beside this one
+  // does (task 1004128). `appUser()` is defined as "the unprivileged account that owns this
+  // checkout" and throws rather than return root, so a runner executing as root reads the
+  // files through the account that may actually read them instead of tripping git's
+  // ownership check. Unprivileged runners are unchanged — there is nothing to escalate to.
+  let runAs = null;
+  if (deps.privileged) {
+    try { runAs = (deps.appUser || appUser)(); } catch { runAs = null; }
+  }
+  const git = (dir, args) => {
+    const r = exec(`${runAs ? `sudo -u ${runAs} ` : ''}git -C ${dir} ${args}`, { allowFail: true });
+    return {
+      ok: !!(r && r.softFailed !== true && r.ok !== false),
+      stdout: String((r && r.stdout) || ''),
+      stderr: String((r && r.stderr) || ''),
+    };
+  };
+  const found = findCoreCheckout(deps, git);
+  if (found.error) return { error: found.error };
+  const dir = found.dir;
+  const shaOf = (v) => {
+    const r = git(dir, `log --format=%H --max-count=1 -F --grep='release: core ${v} '`);
+    if (!r.ok) return { fault: gitFault(r, dir) };
+    const sha = r.stdout.trim().split('\n')[0] || '';
+    return /^[0-9a-f]{7,40}$/.test(sha) ? { sha } : {};
+  };
+  const from = shaOf(fromVersion);
+  const to = shaOf(toVersion);
+  if (from.fault || to.fault) return { error: from.fault || to.fault };
+  // LOOKED, AND DID NOT FIND — a different sentence from "could not look", and it names
+  // WHICH version is missing so the reader can tell a not-yet-published target from a
+  // checkout that has not been fetched.
+  const missing = [!from.sha && fromVersion, !to.sha && toVersion].filter(Boolean);
+  if (missing.length) {
+    return { error: `core ${missing.join(' and ')} ${missing.length > 1 ? 'are' : 'is'} not in the core history on this machine yet` };
+  }
+  const r = git(dir, `log --merges --format=%s%x1f%b%x1e ${from.sha}..${to.sha}`);
+  if (!r.ok) return { error: gitFault(r, dir) };
+  return { entries: parseChangeRecords(r.stdout) };
+}
+
+/**
+ * What landed between two core releases, as the summaries their ships wrote.
+ *
+ * Returns `{ from, to, entries, more, source }`, or `{ from, to, error }` — NEVER an empty
+ * `entries` for a range it could not resolve. An empty list reads as "nothing changed",
+ * which on a page whose job is to say what a restart buys is the one wrong impression it
+ * must not give, and it is exactly what a missing checkout produces.
+ *
+ * THE PACKAGE'S NOTES FIRST, THE CHECKOUT SECOND (task 1004218). The notes answer on every
+ * box; the checkout only where one exists, which is never a tenant. When neither can, the
+ * error carries BOTH reasons — the notes' and the checkout's own precise sentence (task
+ * 1004212) — so the reader is told everything that was tried and why each fell short.
+ *
+ * Every version reaching a shell is re-checked against the strict three-number form
+ * FIRST. They arrive from an intent row whose column has the same CHECK, but a value that
+ * is interpolated into a command is checked where it is interpolated — the rule
+ * upgradeInvocation states about the slug.
+ */
+function changesBetween(fromVersion, toVersion, deps) {
+  const bad = [fromVersion, toVersion].find((v) => !CORE_VERSION_RE.test(String(v || '')));
+  if (bad !== undefined) return { from: fromVersion || null, to: toVersion || null, error: 'could not work out which releases this covers' };
+  const listed = (all, source) => ({
+    from: fromVersion, to: toVersion, entries: all.slice(0, CHANGE_CAP), more: Math.max(0, all.length - CHANGE_CAP), source,
+  });
+  const notes = changesFromNotes(fromVersion, toVersion, deps);
+  if (notes.entries) return listed(notes.entries, 'release-notes');
+  const checkout = changesFromCheckout(fromVersion, toVersion, deps);
+  if (checkout.entries) return listed(checkout.entries, 'checkout');
+  return { from: fromVersion, to: toVersion, error: `${notes.why}; ${checkout.error}` };
+}
+
+// Run a dry run for one instance and cache its verdict — the ON-DEMAND leg, drained from
+// a `core-upgrade-preflight` intent the owner's arming queues (task 1004121). The tick
+// does not do this: a verdict that is only ever produced on a background schedule is
+// missing at exactly the moment it is wanted (a fresh release, a new instance), and
+// re-running it every tick for a box that has been behind for hours spends a subprocess
+// to re-learn the same answer. Asked for, then cached, is both cheaper and timelier.
+async function preflightCoreUpgrade(inst, deps, intent) {
+  const { db, log } = deps;
+  const to = intent && intent.target_version;
+  if (!UPGRADE_SHAPES.has(inst.hosting_shape) || inst.status !== 'active') {
+    return { ok: false, error: `cannot preview a ${inst.hosting_shape} instance` };
+  }
+  if (!to) return { ok: false, error: 'no target version to preview' };
+  const verdict = (deps.preflightUpgrade || preflightUpgrade)(inst, to, deps);
+  // AND WHAT THE MOVE WOULD CARRY, taken in the same breath (task 1004173). It rides the
+  // verdict rather than a column of its own for the reason the verdict's own `target`
+  // exists: both answers are about ONE move, and storing them together is what makes
+  // "is this still the move on offer" a single question with a single answer. It also
+  // costs the owner no second wait — the dry run is already the thing they are waiting on.
+  //
+  // `from` is the version the box is on RIGHT NOW, read back from the snapshot the tick
+  // writes, and it is stored because the answer is only true of that starting point. The
+  // web tier withholds the list when the box has moved since, the same way it withholds a
+  // verdict taken against a superseded target.
+  let servedNow = null;
+  try {
+    const { rows } = await db.query('SELECT served_version FROM provisioning_core_version_snapshot WHERE instance_id = $1', [inst.id]);
+    servedNow = (rows[0] && rows[0].served_version) || null;
+  } catch { /* no snapshot yet — changesBetween refuses on the missing version, and says so */ }
+  verdict.changes = (deps.changesBetween || changesBetween)(servedNow, to, deps);
+  await db.query(
+    `INSERT INTO provisioning_core_version_snapshot (instance_id, preflight, preflight_at, observed_at)
+     VALUES ($1, $2::jsonb, now(), now())
+     ON CONFLICT (instance_id) DO UPDATE
+       SET preflight = EXCLUDED.preflight, preflight_at = now()`,
+    [inst.id, JSON.stringify(verdict)]);
+  log(`  ${verdict.ok ? '✓' : '✖'} ${inst.slug}: pre-flight for ${to} — ${verdict.ok ? 'clear' : verdict.error}`);
+  // ok:true means THE PREVIEW RAN. A refusal is a successful preview of a refusal — the
+  // intent must not go to 'error', or the owner's next arming is blocked by a stale open
+  // intent describing a check that worked perfectly.
+  return { ok: true, preflight: verdict };
+}
+
+/**
+ * Observe one instance and cache what the deploy control needs, without acting.
+ *
+ * Runs on every drain tick for each upgradable instance. Both facts it records are ones
+ * the web tier cannot reach for itself — the served version lives on loopback, the
+ * available versions need a blocking `npm view` (migration provisioning_019 says why at length)
+ * — so this is the runner reporting on behalf of a process that may only ask.
+ *
+ * BEST-EFFORT, ALWAYS. A registry hiccup or a box that will not answer must never fail a
+ * tick that has real work in it: the snapshot simply keeps its previous value and its
+ * previous, now-visibly-older, observed_at. That staleness is the honest signal, and it
+ * is why observed_at is stored rather than assumed.
+ */
+async function snapshotCoreVersions(inst, deps) {
+  const { db, exec, log } = deps;
+  if (!UPGRADE_SHAPES.has(inst.hosting_shape) || inst.status !== 'active') return { ok: true, skipped: true };
+  const boxExec = inst.hosting_shape === 'cloud-host' ? (deps.controlExec || exec) : exec;
+  const channelMod = deps.updateChannel || require('./update-channel.js');
+  const selfRoot = deps.selfInstanceRoot || selfInstanceRoot;
+  const instanceDir = inst.hosting_shape === 'cloud-host' ? standaloneRoot(inst) : selfRoot();
+  try {
+    const probe = boxExec(versionCmd(inst), { allowFail: true });
+    const body = probe && !probe.softFailed ? (probe.stdout || '') : null;
+    const served = parseServedVersion(body);
+    const schemaPending = parseSchemaPending(body);
+    const listed = offeredVersions(channelMod, inst, instanceDir);
+    const available = (listed && listed.versions) || [];
+    // What the channel would take FROM the version the box is actually on. Resolving it
+    // here rather than in the browser is what keeps the button's promise and the
+    // runner's behaviour the same sentence.
+    // resolveChannelTarget returns the bare version string, or null when the channel
+    // admits nothing newer — "already current" and "nothing to take" are the same
+    // answer here, and both correctly leave `recommended` NULL.
+    const recommended = served
+      ? channelMod.resolveChannelTarget({ installed: served, channel: instanceChannel(inst, channelMod), available })
+      : null;
+    // THE TICK DOES NOT DRY-RUN. The verdict is produced on demand (preflightCoreUpgrade),
+    // because a check re-run every tick for a box that has been behind for hours spends a
+    // subprocess to re-learn the same answer, and is still missing in the one window that
+    // matters — the minutes after a release, which is when someone presses the button.
+    // The version probe below is cheap and genuinely per-tick; a dry run is not.
+    const preflight = null;
+    await db.query(
+      `INSERT INTO provisioning_core_version_snapshot
+         (instance_id, served_version, available, recommended, observed_at, preflight, preflight_at, schema_pending)
+       VALUES ($1, $2, $3::jsonb, $4, now(), $5::jsonb, $6, $7)
+       ON CONFLICT (instance_id) DO UPDATE
+         SET served_version = EXCLUDED.served_version, available = EXCLUDED.available,
+             recommended = EXCLUDED.recommended, observed_at = now(),
+             schema_pending = EXCLUDED.schema_pending,
+             -- A tick with nothing to preview must not ERASE the last verdict; it simply
+             -- has nothing newer to say. Only a real dry run overwrites one.
+             preflight = COALESCE(EXCLUDED.preflight, provisioning_core_version_snapshot.preflight),
+             preflight_at = COALESCE(EXCLUDED.preflight_at, provisioning_core_version_snapshot.preflight_at)`,
+      [inst.id, served, JSON.stringify(available), recommended || null,
+       preflight ? JSON.stringify(preflight) : null, preflight ? new Date() : null, schemaPending]);
+    return { ok: true, served, available: available.length, preflight, schemaPending };
+  } catch (e) {
+    log(`  ~ core-version snapshot for ${inst.slug} skipped (${(e && e.message) || e})`);
+    return { ok: true, skipped: true };
+  }
+}
+
+/** Snapshot every instance whose box the runner can reach. Never throws. */
+async function snapshotAllCoreVersions(deps) {
+  const { db, log } = deps;
+  const shapes = [...UPGRADE_SHAPES];
+  const { rows } = await db.query(
+    `SELECT * FROM provisioning_instances WHERE status = 'active' AND hosting_shape = ANY($1)`, [shapes]);
+  let seen = 0;
+  for (const inst of rows) {
+    const r = await snapshotCoreVersions(inst, deps);
+    if (r && !r.skipped) seen++;
+  }
+  if (seen) log(`core-version snapshot: refreshed ${seen} instance(s)`);
+  return { seen };
+}
+
+module.exports = { pushUpgradePin, UPGRADE_SHAPES, instanceChannel, CHANGE_CAP, REPO_NOTES, repoNote, requestedByOf, parseChangeRecords, readInstalledNotes, readPackagedNotes, changesBetween, coreCheckoutCandidates, findCoreCheckout, gitFault, resolveRunAs, versionCmd, parseServedVersion, parseSchemaPending, targetFault, parsePreflight, upgradeInvocation, preflightUpgrade, preflightCoreUpgrade, coreUpgradeInstance, snapshotCoreVersions, snapshotAllCoreVersions };