@bongos/core 1.19.679 → 1.19.680

package/.bongos-core.json

@@ -2,22 +2,22 @@
   "artifact": "bongos-core",
   "manifest_schema": 1,
   "generator": "scripts/gds/package-core.js",
-  "core_version": "1.19.679",
-  "core_contract": "1.19.679",
-  "source_commit": "f1aaf763da799cb5f4ab969af528d5f74c2433c9",
+  "core_version": "1.19.680",
+  "core_contract": "1.19.680",
+  "source_commit": "e022c08bc3285327348cd482d38e0659e2ae8b3d",
   "source_ref": "HEAD",
-  "built_at": "2026-09-11T23:20:02.319Z",
+  "built_at": "2026-09-12T01:35:27.332Z",
   "redaction": {
     "model": "docs-redacted+functional-verbatim",
-    "docs_redacted": 474,
+    "docs_redacted": 476,
     "agent_docs_stubbed": 24,
     "functional_verbatim": 2124,
     "rules": 3,
     "gate_literals": 3,
     "gate": "passed"
   },
-  "file_count": 2622,
-  "tree_sha256": "0de186e89d491b293f3a8acef2deb589062a456c992a3a9ea94f80a7df5c776c",
+  "file_count": 2624,
+  "tree_sha256": "912d1481802f12085195156ce06ab53a39ca1e159441ebee3efe401ddbedf56d",
   "files": [
     {
       "path": ".claude/skills/ask-for-help/SKILL.md",
@@ -1904,10 +1904,15 @@
       "mode": "0000644",
       "sha256": "3d941c0e1f1ce5618fa6861a720a4ac1c1988495c5f17aed77a348a4a3b7b718"
     },
+    {
+      "path": "docs/adr/0279-an-upgrade-is-proven-by-the-served-version-not-the-health-check.md",
+      "mode": "0000644",
+      "sha256": "46e65d06711ad356563b01b824c416d03cecf135f9f69a60e469710e91cb3f62"
+    },
     {
       "path": "docs/adr/README.md",
       "mode": "0000644",
-      "sha256": "d9dcea00e72214a902f4c4b49f996645495df14334eea2b4baf53af5e55dbcc2"
+      "sha256": "6766d2b792da65e6cebc79adf4261b710d6f5ce113a8a84953cf1c643e20a0b7"
     },
     {
       "path": "docs/api-reference.md",
@@ -2792,7 +2797,7 @@
     {
       "path": "docs/module-api-changelog.md",
       "mode": "0000644",
-      "sha256": "462f8656b5bcf15070e7cd7d711e05bdf39bb05c2217352217d7ccdfa24ff776"
+      "sha256": "592be55bbe17442266b4478ca962fcc0bdc4cbd66718c6b4af13a1cca911cc3e"
     },
     {
       "path": "docs/modules-contract.md",
@@ -2949,6 +2954,11 @@
       "mode": "0000644",
       "sha256": "259148d6e612209502b3f46e82e57c62da049fac4d883755e151be37de92fbf7"
     },
+    {
+      "path": "docs/recipes/instance-service-restart.md",
+      "mode": "0000644",
+      "sha256": "77b4f5e695ee839b6e9f7e192af6f13114069a114cc2f63167582a4a8768f4ce"
+    },
     {
       "path": "docs/recipes/local-dev.md",
       "mode": "0000644",
@@ -7767,12 +7777,12 @@
     {
       "path": "package-lock.json",
       "mode": "0000644",
-      "sha256": "673d8ec9856db8cea795601e8055b820af45c0fbabd3694c72585efd1ca74ea6"
+      "sha256": "f0fd6366abe9523c64332de417406bfe0ab6f6872abc6e651debbe5f1941cad0"
     },
     {
       "path": "package.json",
       "mode": "0000644",
-      "sha256": "23342005df25520523dd82a3991379e717b7146630e73564006381cf458477a8"
+      "sha256": "60cff4653b02fd38c3c7fb8673cb423a57f8360955578a6a345ccaef9bf1ac9a"
     },
     {
       "path": "public-docs/index.html",
@@ -9102,7 +9112,7 @@
     {
       "path": "scripts/gds/upgrade.js",
       "mode": "0000644",
-      "sha256": "7a173ecf47b25007bba28f68787893352f429a52b34b7f7b4eae0d3791a5cdcf"
+      "sha256": "642d0ade55c203d6b07478a07d3c21c26fa4635dc8a92e4fe989434e635abbbb"
     },
     {
       "path": "scripts/gds/validate-design.js",
@@ -9532,7 +9542,7 @@
     {
       "path": "src/module-api.js",
       "mode": "0000644",
-      "sha256": "90415725c895791b66d730f3d6d25988f76e748d2509ec2db13915e64c0fc163"
+      "sha256": "19014335bfaede6fba2e6ff79ba1fe746a2149345e25448b2ed0176bdb8a6649"
     },
     {
       "path": "src/module-loader/catalog.js",
@@ -12977,7 +12987,7 @@
     {
       "path": "tests/upgrade.mjs",
       "mode": "0000644",
-      "sha256": "3c7203cb85a187ce5ff0cff16ffd3dc1e02256501318b7fbb6e273e67abc2e99"
+      "sha256": "106735a9ab28b4a23e2f25269030d982798da3fa39e31d40a1ec78277ac971bc"
     },
     {
       "path": "tests/upgrade_persist_pin.mjs",

package/docs/adr/0279-an-upgrade-is-proven-by-the-served-version-not-the-health-check.md

@@ -0,0 +1,48 @@
+# ADR 0279 — An upgrade is proven by the served version, not by a health check
+
+- **Status:** accepted
+- **Date:** 2026-09-11
+- **Task:** [task 1002884](https://cloudbongos.com/builders#/task/1002884) (BONGOS-V2, goal 1000090 — *Working area 4, Bongos Core distribution*), from idea 1000682
+- **Extends** the mandatory post-restart health check of task 1002222 (audit H9, finding F5) and the auto-rollback of task 2149, whose unattended policy is [ADR 0136](<redacted>.md) §3. The health check stays; this says why it was never sufficient on its own.
+
+## Context
+
+On 2026-08-11 the auto-upgrade sweep on cloudbongos.com printed `✓ upgrade complete — core 1.19.13 → 1.19.56`, wrote a success row to the `core_upgrades` ledger and exited 0. Live kept serving **1.19.13**. Every monitoring surface agreed the upgrade had landed. None of them had looked at the running process.
+
+Three things had to line up, and all three were in the shipped code:
+
+1. **The restart failed, and that was a warning.** The sweep ran `systemctl restart cloudbongos.service` as user `lars` with no TTY. The unit is a `User=` unit, so the restart needed authorization polkit could only get interactively — `Interactive authentication required`, exit 1. `upgrade.js` logged `! restart command failed` and **fell through to verification**.
+2. **The version check read the wrong thing.** `readInstalledCoreVersion()` reads `node_modules/@bongos/core/package.json` — the version on **disk**. `npm install` had genuinely put 1.19.56 there. Disk was right; the process was not.
+3. **The health check could not tell the difference.** `pollHealth()` asks whether *something* answers with a 2xx. The old process was up and perfectly healthy, so it answered. A health check confirms a port is being served; it cannot confirm *what* is serving it.
+
+Each check was individually reasonable. Together they were a closed loop that could not observe the failure they existed to catch, which is the worst shape a check can have: it reports success it has not proven, and a lying tool is more expensive than a broken one, because nothing goes looking.
+
+The damage outlived the incident. `.claude/scheduled-tasks/core-update-subscription/subscribe.js` routes around `bongos upgrade` in favour of `go-live.js` and says so in a comment — *"upgrade.js's own health check is documented to false-pass by polling the still-running OLD process"*. A known-lying tool had become something the rest of the system worked around instead of fixing.
+
+## Decision
+
+**A bump is confirmed by asking the running process what version it is. Anything less is unconfirmed, and unconfirmed is not success.**
+
+Three changes in `scripts/gds/upgrade.js`:
+
+1. **A failed restart fails the upgrade.** It enters the same auto-rollback path as a failed install, migrate or health check, and returns non-zero. Rolling back rather than merely erroring keeps disk and process consistent: the previous core goes back on disk, matching the one that is (still) running. Leaving disk ahead of the process is the state that made the original incident invisible.
+
+2. **The served version is read back from `/version`.** That route reports the live process's `coreVersion` (`serve-internal.js`, since 1.17.2) and is exempt from the prelaunch gate, so it is readable wherever `/healthz` is. The URL defaults to `/version` on the health URL's origin and can be overridden with `--version-url`. Deriving rather than requiring a new flag is deliberate: every existing call site passes only `--health-url`, and a check that must be opted into is off exactly where it is needed.
+
+3. **A mismatch fails; an unreadable endpoint warns.** These are different facts and are treated differently. *"It says 1.19.13 and I asked for 1.19.56"* is proof of failure — roll back. *"It would not tell me"* is absence of proof: an instance may not expose the endpoint, and refusing every such bump would be a worse regression than the false-pass we are removing. Passing `--version-url` explicitly asks for proof, so there an unreadable endpoint **is** a failure — which is what the unattended subscription lane now does, since its roster already carries the exact URL.
+
+**On escalation: `sudo -n`, not a hand-placed polkit rule.** `restartService()` retries a failed `systemctl restart` through `sudo -n` when not root — the idiom `dev.js` and `dev-lib.js` already use, and consistent with `provision.js`, which prefixes its own `systemctl` calls with `sudo`. `upgrade.js` was the one place that restarted a service without escalating, which is why only it hit the wall. The live mitigation at the time was a hand-written `/etc/polkit-1/rules.d/<redacted>.rules` on one box; that rule is **superseded**, because a rebuilt box inherits code and does not inherit hand-placed `/etc` files. `-n` fails fast rather than hanging on a password prompt, so the retry costs nothing where it isn't permitted. The polkit route is documented in [`docs/recipes/instance-service-restart.md`](../recipes/instance-service-restart.md) for hosts that prefer it or that deliberately withhold sudo.
+
+## Consequences
+
+- An upgrade that cannot restart the service now **fails loudly and reverts**, where it used to warn and claim success. This is the point, but it means a host with neither passwordless sudo nor a polkit rule will start seeing upgrades fail that previously "passed" — correctly. The error names both fixes.
+- `--no-health-check` remains the single documented escape hatch and now waives the served-version read-back too, rather than adding a second flag to reason about.
+- `subscribe.js` no longer needs to describe the direct-upgrade lane as having "no independent read-back"; the comment and the log line were corrected with the fix, since a stale comment about a fixed bug is how the next session re-learns a lie.
+- The rollback path calls `restartService()` too, so it also escalates — a rollback on a box that needs sudo can now actually restart onto the restored core instead of silently leaving the failure in place.
+
+## Rejected
+
+- **Comparing `startedAt` instead of `coreVersion`.** A restart that swaps nothing still moves `startedAt` if the unit bounced, and a process that never restarted keeps an old one — but `startedAt` cannot distinguish "restarted onto the same (old) core" from "restarted onto the new one". `coreVersion` answers the actual question.
+- **Making an unreadable endpoint fatal by default.** Fails closed in the wrong direction: it would break upgrades on instances that never served `/version`, to catch a case that a fatal restart failure already catches.
+- **Requiring `--version-url` everywhere.** The check would then be absent from every current call site — the same "off where it matters" failure in a new costume.
+- **Keeping the polkit rule as the answer.** It works, but it lives in `/etc` on one machine. The task's own framing is that a rebuilt box must inherit the fix; only code does that.

package/docs/adr/README.md

@@ -370,3 +370,4 @@ This keeps the decision history honest and traceable.
 | 0276 | [**The skill-listing budget cannot hold 64 skills and every trigger, so the number is ratcheted and the choice goes to the owner** ([task 1003620](https://cloudbongos.com/builders#/task/1003620) · goal 1000095 — *Working area 6*, criterion `wa6-role-experience`). Every SKILL.md description is resident in EVERY session for every craft, and `skill-lint` budgets the whole listing at 8,000 chars. It has been trimmed twice ([#1003548], [#1003587]) and grown back both times, because the 400-char per-description aim was a WARNING THAT ACCUMULATES. This task rewrote 49 of the 50 `.claude/skills/` descriptions — 17 warnings to 0, listing 25,624 → 19,316 chars, about 1,570 tokens returned to every session — with every trigger phrase preserved and CHECKED MECHANICALLY (a checker diffed the quoted phrases against HEAD and caught five real losses, all restored; four dropped strings were UI labels, not triggers). **8,000 was not reached, and the arithmetic says it cannot be:** across 64 skills the names (880) plus the mandatory quoted triggers (~5,370) are an irreducible ~6,250, leaving ~27 chars per skill to say what the skill DOES. Reaching the budget means trigger-only descriptions — trading truncation for the loss of the semantic signal routing leans on when the user’s words do not literally match a trigger. **Decision: ratchet `skill_listing_chars` at 19,316** in `fitness-ratchets.js` (fails open if the linter cannot load), so it cannot grow while the real question is open; the baseline is deliberately ABOVE skill-lint’s budget and is not a claim the budget is met. **Open and owner-gated:** delist skills (64 is a menu), scale the budget with the roster, or accept trigger-only text — recommendation is delist then rescale, filed as a blocker. `fitness.js`’s over-budget warning STAYS, as the visible trace of that question. Rejected: raising `LISTING_BUDGET_CHARS` to silence the warning (deletes the signal, not the debt); re-cutting the 14 module ui-design descriptions a week after [#1003587] wrote them, for ~2,000 chars toward a target still 9,000 away.](<redacted>.md) | skills / context budget / routing |
 | 0277 | [**A box is "in use" only while a human is attached, and the claim expires** ([task 1003507](https://cloudbongos.com/builders#/task/1003507) · goal 1000095 — *Working area 6*, criterion `wa6-role-experience`). `sweep-idle` skipped any box with `claude_active = true` at ANY age, and `claude_active` is a LATCH, not a level: only a heartbeat ping writes it, and `infra/box-heartbeat.sh` exits WITHOUT pinging when it sees nothing — so silence, the very signal the sweep exists to act on, could never clear it. Reproduced against the shipped selector: skipped at `idleMinutes` of 10, 120, 1440 and **5,256,000** (ten years). Not reaped late; never. The second leg is that `load > 0.2` kept `last_activity_at` bumping every 5 minutes anyway (a devcontainer plus an idle `claude` clears that floor on its own), so EITHER leg alone kept a box alive — which is why bounding only the veto looks like a fix and is not: the incident box's heartbeat was FRESH. Measured: `example-owner`'s box up since 2026-08-12, tmux `otb` UNATTACHED since 2026-08-15 18:01 UTC, `claude` burning 16 min of CPU across 25h of wall clock, **$20.21** of mostly-unattended compute. A THIRD defect, found while testing this and confirmed on clean `origin/main`, made the sweep inert regardless: `loadDeps()` read `_deps` before anything declared it, so it threw `ReferenceError` on first call and every command resolving a DigitalOcean client went with it — including `sweep-idle --apply`, which builds that client before the park loop. **The idle sweep could not park any box at all**, hidden because the suite's only `apply: true` test relied on the veto emptying the idle set before `makeDo()` was reached: one bug shielded by the other. **Decision: a box may not stay active longer than `BOX_UNATTENDED_MAX_HOURS` (12) without evidence a HUMAN was attached.** The heartbeat already computed that signal and folded it into one boolean; it now reports `attached` separately (login session, inbound SSH, open ttyd, or an ATTACHED tmux client via `#{session_attached}` — the signal that separates this box from a working one) and the server stamps `last_attached_at`. `claude_active` keeps its veto but it EXPIRES, bounded by a `claude_active_since` edge stamp. `pgrep -x claude` and the load floor remain reasons the box PINGS, never evidence anyone is THERE — a running process is not a person, and that distinction is the whole decision. The two `NULL` defaults deliberately DISAGREE: `last_attached_at` NULL means "no data" and keeps a pre-1003507 box on the old behaviour (core_238 pointedly does NOT backfill it — a backfilled `now()` starts a clock nothing can advance and parks every un-upgraded box one cap later, and box source sync is not prompt: idea 1000745 records 257 commits behind for three days), while `claude_active_since` NULL is REFUSED because an unknown latch age is the forever-latch itself, and is backfilled so the state is unreachable after deploy. 12h because parking is reversible since task 1002726 (snapshot kept), so a false positive costs one wake against $20.21 for no cap. Both clocks clear at park/wake/deprovision, or a woken box inherits an expired latch and is parked instantly — the fix reintroducing the bug from the far side. Rejected: tracking attachment INSTEAD of bounding the latch (the silent box keeps `true` forever, so the reported hole survives); bounding the latch alone (built first, and the verification probe caught it — the fresh heartbeat meant lifting the veto changed nothing); deleting the load floor (it is what makes an autonomous run count); measuring from `active_since` (that is uptime — parks a box worked on for days); raising `BOX_IDLE_MINUTES` (no threshold reaches an unbounded veto).](<redacted>.md) | dev box / cost / idle sweep |
 | 0278 | [**A gated project still takes applications, and the exemption is scoped to the verb** ([task 1003525](https://cloudbongos.com/builders#/task/1003525) · the apply write itself in [task 1003624](https://cloudbongos.com/builders#/task/1003624) · goal 1000106 — *Working area 1, Project creation*; owner decision 2026-09-11). A project's owner sets who may SEE it (`platformVisibility`, [ADR 0192](<redacted>.md)) and who may JOIN it (`joinability`, [ADR 0194](<redacted>.md)) independently — and set to their middle values, members-only AND apply-to-join, the project took no applications at all: the member door refused every cookie-less request with `401` before the public `POST <api>/access-requests` could answer, because that write was not on the exempt list. The two settings composed into **"nobody can apply"**, which nobody chose. It survived because nothing LIED about it — the hub's join box relayed the project's own `401` honestly as `members_only`, and the hall's landing, where the apply form lives, is itself behind the door; the composition was simply unreachable. Found by the R14 proof ([task 1002333](https://cloudbongos.com/builders#/task/1002333)). ADR 0192 §3 had fixed the exempt list at "the door, the manifest, the probes and the downloads — and nothing wider" and left widening it as an owner call, which is what this is. **Decision: yes — and BOTH halves are exempted, each scoped to one path and one verb.** `POST <api>/access-requests` (it grants nothing — an application is a row in a queue the owner still reviews, [ADR 0201](<redacted>.md), already public on every non-gated project) and `GET <api>/access-requests/status` (without it the answer is half an answer: `bongos login` cannot re-poll the device flow after a `not_approved` — the `device_code` is spent — so an applicant would file a request and then wait on an approval they can never observe). `EXEMPT` entries may now be `{ re, methods }` beside the bare `RegExp`s, and `isExempt(path, method)` takes the verb as an OPTIONAL second argument that **fails closed** for a scoped entry when none is given, so the one-argument static callers cannot accidentally widen. **The verb is load-bearing, not tidiness:** the bare `GET` on `<api>/access-requests` is the OWNER'S QUEUE (`requireBuilder` + `access_request.review`), the surface listing would-be builders by name with their vouch state — a path-only exemption would have silently taken the member door off the front of it, leaving one layer where there were two, and the queue's own `requirePermission` still holding is exactly what makes that loss easy to miss. **It opens no oracle the gate was closing:** the status route's boolean twin `GET <api>/auth/web/admission-status` is ALREADY reachable on a gated project inside the `auth/*` subtree §3 exempts whole (§3 records that cost in as many words), and the two share ONE per-IP budget on purpose ([ADR 0209](<redacted>.md)) so neither can be alternated against the other. What it DOES add, stated as the honest cost: applicant detail — `pending`/`dismissed`/`none` over the twin's bare `admitted`. Whether that answer should collapse is ADR 0209's still-open owner question and is deliberately NOT decided here. No hub change: `joinRelayOutcome` maps the RELAYED status, so it carries the project's real answer the moment the `401` stops. Rejected: "gated means gated" — hide *Apply to join* and say so in the manage blurb (coherent, and the call went the other way); exempting the path without the verb; exempting the write alone; collapsing the status response while the route happened to be open (that is how a deferred decision gets made by accident).](<redacted>.md) | project visibility / join door / member door |
+| 0279 | [**An upgrade is proven by the served version, not by a health check** ([task 1002884](https://cloudbongos.com/builders#/task/1002884) · goal 1000090 — *Working area 4, Bongos Core distribution*; from idea 1000682). On 2026-08-11 the auto-upgrade sweep printed `✓ upgrade complete — core 1.19.13 → 1.19.56`, wrote a success row to `core_upgrades` and exited 0 while live kept serving **1.19.13**. Three shipped checks formed a closed loop that could not see the failure they existed to catch: the `systemctl restart` failed with `Interactive authentication required` (a `User=` unit, no TTY) and `upgrade.js` treated it as a WARNING and fell through; `readInstalledCoreVersion()` then confirmed the version on **disk**, where `npm install` had correctly put it; and `pollHealth()` got a 200 from the **still-running old process**, because a health check confirms a port is served, never *what* serves it. A lying tool is worse than a broken one — nothing goes looking. The damage outlived the incident: `subscribe.js` had already routed the unattended lane around `bongos upgrade` in favour of `go-live.js`, citing this false-pass in a comment. **Decision: a bump is confirmed by asking the running process what version it is.** (1) A failed restart enters the same auto-rollback path as a failed install/migrate/health and exits non-zero — rolling back rather than merely erroring keeps disk and process consistent, since disk-ahead-of-process is the state that made the incident invisible. (2) The served version is read back from `/version` (`coreVersion`, since 1.17.2, prelaunch-gate exempt), defaulting to the `--health-url` origin and overridable via `--version-url` — derived rather than opt-in because every existing call site passes only `--health-url`, and a check you must opt into is off exactly where it is needed. (3) A **mismatch** fails (proof of failure → roll back); an **unreadable** endpoint only warns (absence of proof — refusing every such bump would regress harder than the false-pass), except under an explicit `--version-url`, which asks for proof and therefore gets a failure. That strict mode is what the unattended subscription lane now passes, its roster already carrying the URL. **Escalation is `sudo -n`, not a hand-placed polkit rule:** `restartService()` retries a failed restart through `sudo -n` when not root — the idiom `dev.js`/`dev-lib.js` already use and consistent with `provision.js`, `upgrade.js` having been the one place that restarted without escalating. The live `/etc/polkit-1/rules.d/<redacted>.rules` mitigation is superseded: a rebuilt box inherits code, not hand-placed `/etc` files (polkit route kept in [`docs/recipes/instance-service-restart.md`](../recipes/instance-service-restart.md)). `--no-health-check` stays the single escape hatch and now waives the read-back too. Rejected: comparing `startedAt` (cannot distinguish a restart onto the same old core from one onto the new); making an unreadable endpoint fatal by default; requiring `--version-url` everywhere (absent from every current call site — the same "off where it matters" failure in a new costume); keeping polkit as the answer.](<redacted>.md) | core distribution / upgrade verification |

package/docs/module-api-changelog.md

@@ -1817,5 +1817,7 @@ is load-bearing: the script throws rather than guess if it is missing, and
         landed since 1.19.677 with no explicit bump. run 34653458123. (task 1002620)
   1.19.679 — CI auto-patch (publish-on-merge, ADR 0161): carrier for merges
         landed since 1.19.678 with no explicit bump. run 34657605820. (task 1002620)
+  1.19.680 — CI auto-patch (publish-on-merge, ADR 0161): carrier for merges
+        landed since 1.19.679 with no explicit bump. run 34665216511. (task 1002620)
 ---------------------------------------------------------------------------
 ```

package/docs/recipes/instance-service-restart.md

@@ -0,0 +1,89 @@
+# Recipe — letting `bongos upgrade` restart the instance service unattended
+
+> [ADR 0279](../adr/<redacted>.md) · [task 1002884](https://cloudbongos.com/builders#/task/1002884). A core bump only counts once the **running process** is the new core, so `bongos upgrade` has to be able to restart the service without a human at a keyboard. This is what to do when it cannot.
+
+## The symptom
+
+```
+! restart command failed (exit 1) — restart the service manually.
+```
+
+or, in the journal for the unit:
+
+```
+Interactive authentication required.
+```
+
+Since task 1002884 this **fails the bump and rolls it back** rather than warning and carrying on. That is the fix working: before it, the run went on to health-check the still-running old process, got a 200, and reported `✓ upgrade complete` for a core that was never served.
+
+## Why it happens
+
+A provisioned instance runs under a `User=` systemd unit. `systemctl restart <unit>` from the owning (non-root) user is a privileged action, so systemd asks polkit, and polkit wants to authenticate a human. In an interactive shell you get a password prompt. In the unattended sweep there is no TTY, so polkit refuses outright and `systemctl` exits 1.
+
+## The fix, in order of preference
+
+### 1. Passwordless sudo (what the code already tries)
+
+`upgrade.js` retries a failed `systemctl restart` through `sudo -n systemctl restart <unit>` when it is not running as root — the same escalation `dev.js`, `dev-lib.js` and `provision.js` use. If the service owner has passwordless sudo, **nothing needs configuring** and the bump succeeds on the retry (the log says `✓ restarted <unit> (via sudo -n)`).
+
+Check whether the owner already has it:
+
+```bash
+sudo -n -l
+```
+
+A `sudo: a password is required` means no. Note the `-n`: without it this command blocks on a prompt, which is exactly the failure being diagnosed.
+
+### 2. Scoped sudoers, if blanket sudo is too much
+
+Grant only the restart, not everything. Write it with `visudo -f` so a syntax error cannot lock the host out:
+
+```bash
+sudo visudo -f /etc/sudoers.d/50-<slug>-restart
+```
+
+```
+<owner> ALL=(root) NOPASSWD: <redacted> restart <slug>.service
+```
+
+Then confirm the retry path works, non-interactively:
+
+```bash
+sudo -n systemctl restart <slug>.service && echo OK
+```
+
+### 3. A polkit rule, for hosts that deliberately withhold sudo
+
+Equivalent outcome, different mechanism — use it when policy forbids a sudoers entry.
+
+```bash
+sudo tee /etc/polkit-1/rules.d/50-<slug>-restart.rules >/dev/null <<'RULE'
+polkit.addRule(function(action, subject) {
+  if (action.id == "org.freedesktop.systemd1.manage-units" &&
+      action.lookup("unit") == "<slug>.service" &&
+      subject.user == "<owner>") {
+    return polkit.Result.YES;
+  }
+});
+RULE
+```
+
+No daemon reload is needed; polkit picks up rules files on the next check.
+
+> **This file is not inherited by a rebuilt box.** It lives in `/etc` on one machine. A reprovision restores code, not hand-placed host config — which is precisely why the `sudo -n` retry lives in the core and this page is the fallback rather than the answer. If you reach for this route, record it in the instance's own provisioning notes or the box will re-discover the stall.
+
+## Verifying it actually worked
+
+Do not trust the exit code alone — that is the failure mode this whole page exists because of. Ask the running process what it is:
+
+```bash
+curl -s https://<instance>/version
+```
+
+`coreVersion` in that response is the live process's core. If it still reads the old version after a "successful" restart, the process did not swap, and `bongos upgrade` will now catch that itself and roll back.
+
+## Related
+
+- [ADR 0279](../adr/<redacted>.md) — why a health check cannot confirm an upgrade
+- [ADR 0136](../adr/<redacted>.md) §3 — the unattended subscription lane's health-gate + auto-rollback policy
+- [`docs/recipes/ops-gotchas.md`](ops-gotchas.md) — the wider set of deploy traps

package/package-lock.json

@@ -1,12 +1,12 @@
 {
   "name": "@bongos/core",
-  "version": "1.19.679",
+  "version": "1.19.680",
   "lockfileVersion": 3,
   "requires": true,
   "packages": {
     "": {
       "name": "@bongos/core",
-      "version": "1.19.679",
+      "version": "1.19.680",
       "license": "AGPL-3.0-or-later",
       "dependencies": {
         "express": "^4.21.2",

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@bongos/core",
-  "version": "1.19.679",
+  "version": "1.19.680",
   "description": "Cloud Bongos — the AI-first build platform core (GDS + platform surfaces + module system), installed as a versioned dependency (ADR 0108).",
   "license": "AGPL-3.0-or-later",
   "main": "src/platform-server.js",

package/scripts/gds/upgrade.js

@@ -521,7 +521,20 @@ function healDocAssets({ coreRoot }, run = spawnSync, fsImpl = fs) {
   return { ok: failed.length === 0, ran, failed, missing };
 }
 
-function restartService({ service, restartCmd }, run = spawnSync) {
+// Restart the instance service so the newly-installed core becomes the one being SERVED.
+// Escalation is the whole difficulty here (task 1002884). A provisioned instance runs under a
+// `User=` systemd unit, and the unattended sweep that restarts it has no TTY — so a bare
+// `systemctl restart` from the owning (non-root) user returns "Interactive authentication
+// required" and exits 1. We retry once through `sudo -n`, the same idiom dev.js and dev-lib.js
+// already use for privileged systemctl; `-n` fails fast rather than blocking on a password
+// prompt, so the retry costs nothing when it isn't permitted. An operator who prefers polkit
+// over sudo can install the rule in docs/recipes/instance-service-restart.md instead — either
+// path makes this return ok.
+// `--restart-cmd` is NOT escalated: an explicit command is the operator's own contract.
+// Trust boundary: `service` comes from `--service` or the subscription roster — operator-supplied
+// config, never request input — and the escalation reaches only as far as the host's own sudoers
+// or polkit scoping allows. Do not wire a caller that derives `service` from anything untrusted.
+function restartService({ service, restartCmd }, run = spawnSync, deps = {}) {
   if (restartCmd) {
     const parts = restartCmd.split(/\s+/);
     const r = run(parts[0], parts.slice(1), { stdio: 'inherit' });
@@ -529,7 +542,14 @@ function restartService({ service, restartCmd }, run = spawnSync) {
   }
   if (service) {
     const r = run('systemctl', ['restart', service], { stdio: 'inherit' });
-    return { ok: !!r && !r.error && r.status === 0, code: r && r.status };
+    if (r && !r.error && r.status === 0) return { ok: true, code: 0 };
+    const isRoot = deps.isRoot !== undefined
+      ? !!deps.isRoot
+      : (typeof process.getuid === 'function' && process.getuid() === 0);
+    if (isRoot) return { ok: false, code: r && r.status };
+    const s = run('sudo', ['-n', 'systemctl', 'restart', service], { stdio: 'inherit' });
+    const ok = !!s && !s.error && s.status === 0;
+    return { ok, code: ok ? 0 : (s && s.status), triedSudo: true, directCode: r && r.status };
   }
   return { ok: null, skipped: true };
 }
@@ -622,6 +642,68 @@ async function pollHealth(healthUrl, deps = {}) {
   }
 }
 
+// ---- served-version verification (task 1002884) -----------------------------
+// The gap this closes: after a restart, `readInstalledCoreVersion` reads node_modules on DISK
+// and `pollHealth` proves only that *something* answers the port. Neither can tell "the new core
+// is up" from "the old core never went down" — so when a restart silently failed on
+// cloudbongos.com (2026-08-11), the health poll hit the STILL-RUNNING old process, got 200, and
+// the bump reported "✓ upgrade complete" while live kept serving the previous core.
+// `/version` reports the LIVE process's coreVersion (serve-internal.js, since 1.17.2), which is
+// the one signal that distinguishes the two.
+
+// Default the version endpoint to /version on the same origin as the health URL. Deriving rather
+// than demanding a second flag is deliberate: every existing call site passes only --health-url,
+// and a check that has to be opted into is off exactly where it is needed. Returns null when the
+// health URL isn't parseable as one.
+function deriveVersionUrl(healthUrl) {
+  try {
+    const u = new URL(healthUrl);
+    u.pathname = '/version';
+    u.search = '';
+    u.hash = '';
+    return u.toString();
+  } catch { return null; }
+}
+
+// Poll the version endpoint until it yields a parseable coreVersion or the budget elapses. Runs
+// AFTER the health poll, so the service is already answering and the budget is short — this is
+// re-reading a live endpoint, not waiting for a boot. A body without a usable coreVersion is
+// "unreadable", NOT a mismatch: the caller must be able to tell "it says the wrong version"
+// (a real failure) from "it wouldn't tell me" (unconfirmed), because only the first is proof.
+async function pollServedVersion(versionUrl, deps = {}) {
+  const fetchImpl = deps.fetch || fetch;
+  const sleep = deps.sleep || ((ms) => new Promise((resolve) => setTimeout(resolve, ms)));
+  const now = deps.now || Date.now;
+  // Default to the health budget when a caller (or a test) has already narrowed it, else 15s.
+  const budgetMs = deps.versionBudgetMs != null
+    ? deps.versionBudgetMs
+    : (deps.healthBudgetMs != null ? deps.healthBudgetMs : 15_000);
+  // A hard attempt cap alongside the clock: `now` is injectable, and a frozen or non-advancing
+  // clock would otherwise spin this loop forever instead of giving up.
+  const maxAttempts = deps.versionMaxAttempts != null ? deps.versionMaxAttempts : 12;
+  const deadline = now() + budgetMs;
+  let attempt = 0;
+  let last = { ok: false, error: 'not attempted' };
+  for (;;) {
+    attempt++;
+    try {
+      const res = await fetchImpl(versionUrl, { signal: AbortSignal.timeout(5_000) });
+      if (!res || !res.ok) last = { ok: false, error: `status ${res && res.status}` };
+      else if (typeof res.json !== 'function') last = { ok: false, error: 'response carried no JSON body' };
+      else {
+        const body = await res.json();
+        const version = body && typeof body.coreVersion === 'string' ? body.coreVersion : null;
+        if (version) return { ok: true, version, attempts: attempt };
+        last = { ok: false, error: 'no coreVersion in the response body' };
+      }
+    } catch (e) {
+      last = { ok: false, error: e.message };
+    }
+    if (now() >= deadline || attempt >= maxAttempts) return { ...last, attempts: attempt, timedOut: true };
+    await sleep(Math.min(2_000, 500 * attempt));
+  }
+}
+
 // ---- integrity-pin verification (task 1002216 / audit H8, finding F1) --------
 // The packager (package-core.js) computes tree_sha256 = canonicalTreeHash over the
 // redacted core files (+ the two synthesized files) and records it, the per-file
@@ -738,7 +820,9 @@ async function rollback({ instanceDir, snapshot, fromVersion, toVersion, opts },
   else err(`  ! reinstall of previous core failed (${inst.reason || 'exit ' + inst.code}) — the instance may need manual repair`);
 
   // 3. restart back onto the previous core.
-  const rs = restartService({ service: opts.service, restartCmd: opts.restartCmd }, run);
+  // `deps` is passed here for the same reason the forward path passes it: this restart runs
+  // AFTER a failure, so its sudo-escalation is the one that matters most and must be testable.
+  const rs = restartService({ service: opts.service, restartCmd: opts.restartCmd }, run, deps);
   if (rs.skipped) log('  • restart: no --service/--restart-cmd — restart the instance service manually to load the reverted core.');
   else if (!rs.ok) err(`  ! restart after rollback failed (exit ${rs.code}) — restart the service manually.`);
   else log(`  ✓ restarted ${opts.service || 'service'} on the previous core`);
@@ -961,10 +1045,24 @@ async function runUpgrade(opts, deps = {}) {
   } else log('  • doc regen skipped (--skip-regen-docs)');
 
   // 6. restart.
-  const rs = restartService({ service: opts.service, restartCmd: opts.restartCmd }, run);
+  const rs = restartService({ service: opts.service, restartCmd: opts.restartCmd }, run, deps);
   if (rs.skipped) log(`  • restart: no --service/--restart-cmd — restart the instance service to load core ${targetVersion}.`);
-  else if (!rs.ok) err(`  ! restart command failed (exit ${rs.code}) — restart the service manually.`);
-  else log(`  ✓ restarted ${opts.service || 'service'}`);
+  else if (!rs.ok) {
+    // task 1002884: a failed restart is an upgrade FAILURE, not a warning. It used to fall
+    // through to the verification below — but the old process is still up, so the health poll
+    // passed against IT and the bump wrote a success row for a core that was never served.
+    // Failing here also keeps disk and process consistent: rollback puts the previous core back
+    // on disk, matching the one that is (still) running.
+    err(`  ✖ restart failed (exit ${rs.code})${rs.triedSudo ? ` — \`systemctl restart\` exited ${rs.directCode} and \`sudo -n systemctl restart\` could not escalate` : ''}`);
+    err('    The previous core is still the one being served. Grant the restart a non-interactive path');
+    err('    (sudoers NOPASSWD or a polkit rule — docs/recipes/instance-service-restart.md), then re-run.');
+    const error = `restart failed (exit ${rs.code}) — the instance is still running the previous core`;
+    if (rollbackOnFailure && snapshot) {
+      const rb = await doRollback('restart failed');
+      return { ok: false, error, fromVersion, toVersion: targetVersion, restartOk: false, rolledBack: true, rollback: rb };
+    }
+    return { ok: false, error, fromVersion, toVersion: targetVersion, restartOk: false };
+  } else log(`  ✓ restarted ${opts.service || 'service'}${rs.triedSudo ? ' (via sudo -n)' : ''}`);
 
   // 7. verify: installed version == target, optional /healthz, ledger row.
   const installed = readInstalledCoreVersion(instanceDir, fsImpl);
@@ -985,12 +1083,48 @@ async function runUpgrade(opts, deps = {}) {
     log('  • no managed restart + no --health-url — new core installed but not confirmed serving; restart the instance + health-check it to complete the bump.');
   }
 
+  // 7b. Confirm the SERVED process actually swapped (task 1002884). Everything above this point
+  //     can be true of an instance still running the OLD core: the pin moved, node_modules holds
+  //     the new version, and /healthz answers 200 — from the process that never went down. Only
+  //     the live coreVersion settles it.
+  //     A definite MISMATCH is a failure (same treatment as a failed health check). Being unable
+  //     to READ the endpoint is not: an instance may not expose it, and refusing every such bump
+  //     would be a worse regression than the false-pass. Passing --version-url explicitly asks
+  //     for proof, so there an unreadable endpoint IS a failure.
+  let servedOk = null;
+  let servedVersion = null;
+  const versionUrl = opts.versionUrl || (opts.healthUrl ? deriveVersionUrl(opts.healthUrl) : null);
+  if (opts.healthCheck === false) {
+    // --no-health-check is the documented opt-out from confirming the bump serves; it covers this.
+  } else if (versionUrl && healthOk !== false) {
+    const sv = await pollServedVersion(versionUrl, deps);
+    servedVersion = sv.version || null;
+    if (sv.ok && servedVersion === targetVersion) {
+      servedOk = true;
+      log(`  ✓ served core version: ${servedVersion} (${versionUrl})`);
+    } else if (sv.ok) {
+      servedOk = false;
+      err(`  ! served core version is ${servedVersion} but the target is ${targetVersion} (${versionUrl})`);
+      err('    The restart did not swap the running process — the old core is still being served.');
+    } else if (opts.versionUrl) {
+      servedOk = false;
+      err(`  ! could not read ${versionUrl} (${sv.error}) — --version-url was given, so this bump cannot be confirmed`);
+    } else {
+      err(`  ! could not read ${versionUrl} (${sv.error}) — this bump is NOT confirmed against the running process.`);
+      err('    Pass --version-url <url> if this instance serves its version elsewhere.');
+    }
+  }
+
   // A failed post-restart health check means the new core is serving broken — auto-revert (task
-  // 2149) instead of recording an "upgraded" row + leaving the instance down. The rollback writes
-  // its OWN rolled_back ledger row, so skip the normal insert below.
-  if (healthOk === false && rollbackOnFailure && snapshot) {
-    const rb = await doRollback('health check failed after restart');
-    return { ok: false, error: `health check failed after restart — rolled back to ${snapshot.prevVersion || 'the previous core'}`, fromVersion, toVersion: targetVersion, versionOk, healthOk: false, rolledBack: true, rollback: rb };
+  // 2149) instead of recording an "upgraded" row + leaving the instance down. A served-version
+  // mismatch means the new core is not serving AT ALL (task 1002884) and reverts the same way.
+  // The rollback writes its OWN rolled_back ledger row, so skip the normal insert below.
+  if ((healthOk === false || servedOk === false) && rollbackOnFailure && snapshot) {
+    const why = healthOk === false
+      ? 'health check failed after restart'
+      : `served core version did not change (still ${servedVersion || 'unconfirmed'}, expected ${targetVersion})`;
+    const rb = await doRollback(why);
+    return { ok: false, error: `${why} — rolled back to ${snapshot.prevVersion || 'the previous core'}`, fromVersion, toVersion: targetVersion, versionOk, healthOk, servedOk, servedVersion, rolledBack: true, rollback: rb };
   }
 
   const ledger = await recordLedger({ fromVersion, toVersion: targetVersion, sourceCommit: opts.sourceCommit, note: opts.note }, deps);
@@ -1010,9 +1144,9 @@ async function runUpgrade(opts, deps = {}) {
     dryRun: opts.dryRun,
   }, run, log, err);
 
-  const ok = versionOk && healthOk !== false;
+  const ok = versionOk && healthOk !== false && servedOk !== false;
   log(ok ? `\n✓ upgrade complete — core ${fromVersion || '(none)'} → ${targetVersion}` : `\n! upgrade finished with warnings — review above.`);
-  return { ok, fromVersion, toVersion: targetVersion, versionOk, healthOk, ledger: ledger.ok, pin };
+  return { ok, fromVersion, toVersion: targetVersion, versionOk, healthOk, servedOk, servedVersion, ledger: ledger.ok, pin };
 }
 
 function parseArgs(argv) {
@@ -1023,6 +1157,7 @@ function parseArgs(argv) {
     service: arg('--service', argv),
     restartCmd: arg('--restart-cmd', argv),
     healthUrl: arg('--health-url', argv),
+    versionUrl: arg('--version-url', argv), // task 1002884: read the LIVE process's coreVersion back; defaults to /version on the health URL's origin
     sourceCommit: arg('--source-commit', argv),
     note: arg('--note', argv),
     pinPath: arg('--pin-path', argv),
@@ -1058,7 +1193,8 @@ async function main(argv = process.argv.slice(2)) {
       '  --service <unit>     systemd unit to restart after applying',
       '  --restart-cmd <cmd>  explicit restart command (overrides --service)',
       '  --health-url <url>   poll after restart (~30s) until 2xx; a still-failing check fails the bump (auto-rollback). REQUIRED for a bump that restarts the service, unless --no-health-check.',
-      '  --no-health-check    explicitly skip the post-restart health confirmation (default: the check is ON — a bump that restarts the service must confirm it serves)',
+      '  --version-url <url>  read the RUNNING process\'s coreVersion back after restart and require it to equal --to (default: /version on the --health-url origin). A health check alone cannot tell a new core from an old one that never went down.',
+      '  --no-health-check    explicitly skip the post-restart health confirmation (default: the check is ON — a bump that restarts the service must confirm it serves); also skips the served-version read-back',
       '  --skip-migrate       do not run `npm run migrate`',
       '  --skip-materialize   do not refresh .claude/',
       '  --skip-regen-docs    do not regenerate the OpenAPI spec + typed client from the new core',
@@ -1069,8 +1205,8 @@ async function main(argv = process.argv.slice(2)) {
       '  --dry-run            show the plan; change nothing',
       '  --force              skip the clean-tree + same-version guards',
       '',
-      'Full bump: pre-flight → bump pin → npm install → verify integrity pin → migrate → materialize .claude → regen API docs+client → restart → verify.',
-      'On a failure after the pin moves (install/pin-verify/migrate/health), auto-rollback reverts the pin, reinstalls the previous core, restarts, and records a rolled_back ledger row (the DB is not rolled back — core migrations are additive).',
+      'Full bump: pre-flight → bump pin → npm install → verify integrity pin → migrate → materialize .claude → regen API docs+client → restart → verify (health + served coreVersion).',
+      'On a failure after the pin moves (install/pin-verify/migrate/restart/health/served-version), auto-rollback reverts the pin, reinstalls the previous core, restarts, and records a rolled_back ledger row (the DB is not rolled back — core migrations are additive).',
     ].join('\n') + '\n');
     return 0;
   }
@@ -1087,7 +1223,7 @@ async function main(argv = process.argv.slice(2)) {
 module.exports = {
   runUpgrade, parseArgs, main, rollback,
   versionFromTgzPath, versionFromDep, readInstalledCoreVersion, readPinnedCoreVersion, writePin, writeRegistryPin, restorePin,
-  vendorTarball, gitTreeClean, persistPin, dirtyPinFiles, gitCurrentBranch, PIN_FILES, preflightDbIdentity, resolveInstanceDb, preflightModules, reportModulePreflight, npmInstall, runMigrate, regenerateApiArtifacts, regenerateNavDocs, NAV_WHOLE_FILE_GENERATORS, missingDocAssets, healDocAssets, DOC_ASSET_GENERATORS, restartService, recordLedger, pollHealth,
+  vendorTarball, gitTreeClean, persistPin, dirtyPinFiles, gitCurrentBranch, PIN_FILES, preflightDbIdentity, resolveInstanceDb, preflightModules, reportModulePreflight, npmInstall, runMigrate, regenerateApiArtifacts, regenerateNavDocs, NAV_WHOLE_FILE_GENERATORS, missingDocAssets, healDocAssets, DOC_ASSET_GENERATORS, restartService, recordLedger, pollHealth, deriveVersionUrl, pollServedVersion,
   readManifest, resolveReferenceManifest, verifyInstalledPin, // task 1002216 (audit H8) — integrity-pin verification
   CORE_PKG, ARTIFACT,
 };

package/src/module-api.js

@@ -71,7 +71,7 @@ const { responsibilityFor, ROLE_RESPONSIBILITIES } = require('./role-responsibil
 // there. scripts/gds/bump-version.js still rewrites the literal below; it appends
 // the entry to that file. Look for a version's history there, not here.
 // ---------------------------------------------------------------------------
-const CORE_VERSION = '1.19.679'; // CI auto-patch carrier (ADR 0161); changelog: docs/module-api-changelog.md
+const CORE_VERSION = '1.19.680'; // CI auto-patch carrier (ADR 0161); changelog: docs/module-api-changelog.md
 
 // A namespaced logger so a module's log lines are attributable + consistent.
 // Usage: const log = api.logger('dev-box'); log.info('mounted');

package/tests/upgrade.mjs

@@ -1091,6 +1091,265 @@ t('runUpgrade: refuses BEFORE the pin moves when the instance names no database'
   assert.match(readFileSync(join(dir, 'package.json'), 'utf8'), /"@bongos\/core": "1\.0\.0"/);
 });
 
+// ---- task 1002884: an upgrade may not declare success it has not proven -----
+// The 2026-08-11 cloudbongos.com incident, as executable tests. The sweep's `systemctl restart`
+// failed with "Interactive authentication required", upgrade.js logged it as a warning and
+// carried on, the health poll hit the STILL-RUNNING old process and got 200, and the run
+// reported "✓ upgrade complete — core 1.19.13 → 1.19.56" while live served 1.19.13.
+// Both halves are covered: a failed restart must be fatal, and a served version that did not
+// change must be caught even when the restart reported success.
+
+// A run shim whose `systemctl restart` fails, optionally letting `sudo -n systemctl restart` win.
+function restartFailingRun(calls, { dir, sudoWorks = false } = {}) {
+  const base = pinAwareRun(calls, { dir });
+  return (cmd, args = []) => {
+    if (cmd === 'systemctl' && args[0] === 'restart') { calls.push(`systemctl ${args.join(' ')}`); return { status: 1 }; }
+    if (cmd === 'sudo') { calls.push(`sudo ${args.join(' ')}`); return { status: sudoWorks ? 0 : 1 }; }
+    return base(cmd, args);
+  };
+}
+
+// A /version endpoint that reports whatever the live process is "serving".
+function versionFetch(serving, { healthOk = true } = {}) {
+  return async (url) => {
+    if (String(url).endsWith('/version')) return { ok: true, status: 200, json: async () => ({ coreVersion: serving }) };
+    return { ok: healthOk, status: healthOk ? 200 : 503 };
+  };
+}
+
+t('restartService: a non-root systemctl failure retries through `sudo -n` and can succeed there', () => {
+  const calls = [];
+  const run = (cmd, args = []) => { calls.push(`${cmd} ${args.join(' ')}`); return { status: cmd === 'sudo' ? 0 : 1 }; };
+  const r = u.restartService({ service: 'demo.service' }, run, { isRoot: false });
+  assert.equal(r.ok, true, 'the sudo -n retry restarted the service');
+  assert.equal(r.triedSudo, true);
+  assert.equal(r.directCode, 1, 'the direct attempt is reported alongside');
+  assert.deepEqual(calls, ['systemctl restart demo.service', 'sudo -n systemctl restart demo.service']);
+});
+
+t('restartService: as root there is no sudo retry (nothing to escalate to)', () => {
+  const calls = [];
+  const run = (cmd, args = []) => { calls.push(`${cmd} ${args.join(' ')}`); return { status: 1 }; };
+  const r = u.restartService({ service: 'demo.service' }, run, { isRoot: true });
+  assert.equal(r.ok, false);
+  assert.ok(!r.triedSudo, 'root does not shell out to sudo');
+  assert.deepEqual(calls, ['systemctl restart demo.service']);
+});
+
+t('restartService: an explicit --restart-cmd is never escalated (the operator owns that contract)', () => {
+  const calls = [];
+  const run = (cmd, args = []) => { calls.push(`${cmd} ${args.join(' ')}`); return { status: 1 }; };
+  const r = u.restartService({ restartCmd: 'deploy restart' }, run, { isRoot: false });
+  assert.equal(r.ok, false);
+  assert.deepEqual(calls, ['deploy restart'], 'no sudo retry for an explicit command');
+});
+
+t('the 2026-08-11 incident: a failed restart FAILS the bump even though /healthz answers 200', async () => {
+  const src = mkdtempSync(join(tmpdir(), 'src-'));
+  writeFileSync(join(src, 'bongos-core-1.15.0.tgz'), 'TGZ');
+  const dir = scratchConsumer({ pinned: '1.14.1', installed: '1.14.1' });
+  const calls = [], queries = [];
+  const res = await u.runUpgrade(
+    { to: '1.15.0', from: join(src, 'bongos-core-1.15.0.tgz'), instance: dir, service: 'demo.service', healthUrl: 'http://x/healthz' },
+    {
+      log: () => {}, err: () => {},
+      run: restartFailingRun(calls, { dir, sudoWorks: false }),
+      isRoot: false,
+      materialize: () => ({ skills: 1 }), regenerateApiArtifacts: () => ({ ok: true, ran: [] }),
+      databaseUrl: 'postgres://x/y', pg: recordingPg(queries),
+      // the old process is still up and healthy — the exact false-pass this task exists to kill
+      fetch: async () => ({ ok: true, status: 200 }),
+      sleep: async () => {}, now: () => 0, healthBudgetMs: 0,
+    }
+  );
+  assert.equal(res.ok, false, 'a failed restart is not a successful upgrade');
+  assert.equal(res.restartOk, false);
+  assert.match(res.error, /restart failed/i);
+  assert.match(res.error, /still running the previous core/i);
+  assert.equal(res.rolledBack, true, 'the bump reverts rather than leaving disk ahead of the process');
+  assert.equal(u.readPinnedCoreVersion(dir), '1.14.1', 'pin reverted');
+  assert.equal(queries.length, 1, 'no phantom "upgraded" ledger row');
+  assert.match(queries[0].params[3], /^rolled_back: restart failed/);
+  assert.ok(calls.includes('sudo -n systemctl restart demo.service'), 'it tried to escalate before giving up');
+});
+
+t('a restart that fails directly but succeeds under sudo -n carries the bump through', async () => {
+  const src = mkdtempSync(join(tmpdir(), 'src-'));
+  writeFileSync(join(src, 'bongos-core-1.15.0.tgz'), 'TGZ');
+  const dir = scratchConsumer({ pinned: '1.14.1', installed: '1.14.1' });
+  const calls = [], queries = [];
+  const res = await u.runUpgrade(
+    { to: '1.15.0', from: join(src, 'bongos-core-1.15.0.tgz'), instance: dir, service: 'demo.service', healthUrl: 'http://x/healthz' },
+    {
+      log: () => {}, err: () => {},
+      run: restartFailingRun(calls, { dir, sudoWorks: true }),
+      isRoot: false,
+      materialize: () => ({ skills: 1 }), regenerateApiArtifacts: () => ({ ok: true, ran: [] }),
+      databaseUrl: 'postgres://x/y', pg: recordingPg(queries),
+      fetch: versionFetch('1.15.0'),
+      sleep: async () => {}, now: () => 0, healthBudgetMs: 0,
+    }
+  );
+  assert.equal(res.ok, true, 'escalating is a success, not a warning');
+  assert.equal(res.servedOk, true);
+  assert.equal(res.servedVersion, '1.15.0');
+  assert.equal(queries.length, 1);
+  assert.ok(!/rolled_back/.test(queries[0].params[3] || ''), 'a real upgraded row, not a reversal');
+});
+
+t('served-version mismatch: restart reports success but the OLD core is still serving → fail + rollback', async () => {
+  const src = mkdtempSync(join(tmpdir(), 'src-'));
+  writeFileSync(join(src, 'bongos-core-1.15.0.tgz'), 'TGZ');
+  const dir = scratchConsumer({ pinned: '1.14.1', installed: '1.14.1' });
+  const calls = [], queries = [];
+  const res = await u.runUpgrade(
+    { to: '1.15.0', from: join(src, 'bongos-core-1.15.0.tgz'), instance: dir, service: 'demo.service', healthUrl: 'http://x/healthz' },
+    {
+      log: () => {}, err: () => {},
+      run: pinAwareRun(calls, { dir }), // systemctl exits 0 — nothing else can catch this
+      materialize: () => ({ skills: 1 }), regenerateApiArtifacts: () => ({ ok: true, ran: [] }),
+      databaseUrl: 'postgres://x/y', pg: recordingPg(queries),
+      fetch: versionFetch('1.14.1'), // the process never swapped
+      sleep: async () => {}, now: () => 0, healthBudgetMs: 0,
+    }
+  );
+  assert.equal(res.ok, false, 'disk-and-health agreement is not proof the new core is serving');
+  assert.equal(res.servedOk, false);
+  assert.equal(res.servedVersion, '1.14.1');
+  assert.equal(res.rolledBack, true);
+  assert.equal(queries.length, 1, 'no phantom "upgraded" row');
+  assert.match(queries[0].params[3], /^rolled_back: served core version did not change/);
+});
+
+t('served-version match: the bump is confirmed against the running process', async () => {
+  const src = mkdtempSync(join(tmpdir(), 'src-'));
+  writeFileSync(join(src, 'bongos-core-1.15.0.tgz'), 'TGZ');
+  const dir = scratchConsumer({ pinned: '1.14.1', installed: '1.14.1' });
+  const urls = [];
+  const res = await u.runUpgrade(
+    { to: '1.15.0', from: join(src, 'bongos-core-1.15.0.tgz'), instance: dir, service: 'demo.service', healthUrl: 'http://x/healthz' },
+    {
+      log: () => {}, err: () => {},
+      run: pinAwareRun([], { dir }),
+      materialize: () => ({ skills: 1 }), regenerateApiArtifacts: () => ({ ok: true, ran: [] }),
+      fetch: async (url) => { urls.push(String(url)); return versionFetch('1.15.0')(url); },
+      sleep: async () => {}, now: () => 0, healthBudgetMs: 0,
+    }
+  );
+  assert.equal(res.ok, true);
+  assert.equal(res.servedOk, true);
+  assert.ok(urls.includes('http://x/version'), 'the version URL is derived from the health URL origin');
+});
+
+t('an unreadable DERIVED version endpoint warns but does not fail the bump (no regression)', async () => {
+  const src = mkdtempSync(join(tmpdir(), 'src-'));
+  writeFileSync(join(src, 'bongos-core-1.15.0.tgz'), 'TGZ');
+  const dir = scratchConsumer({ pinned: '1.14.1', installed: '1.14.1' });
+  const warnings = [];
+  const res = await u.runUpgrade(
+    { to: '1.15.0', from: join(src, 'bongos-core-1.15.0.tgz'), instance: dir, service: 'demo.service', healthUrl: 'http://x/healthz' },
+    {
+      log: () => {}, err: (m) => warnings.push(String(m)),
+      run: pinAwareRun([], { dir }),
+      materialize: () => ({ skills: 1 }), regenerateApiArtifacts: () => ({ ok: true, ran: [] }),
+      fetch: async (url) => (String(url).endsWith('/version') ? { ok: false, status: 404 } : { ok: true, status: 200 }),
+      sleep: async () => {}, now: () => 0, healthBudgetMs: 0,
+    }
+  );
+  assert.equal(res.ok, true, 'an instance that does not serve /version can still be upgraded');
+  assert.equal(res.servedOk, null, 'unconfirmed is neither pass nor fail');
+  assert.ok(warnings.some((w) => /NOT confirmed against the running process/.test(w)), 'but it says so loudly');
+});
+
+t('an unreadable EXPLICIT --version-url fails the bump (proof was asked for)', async () => {
+  const src = mkdtempSync(join(tmpdir(), 'src-'));
+  writeFileSync(join(src, 'bongos-core-1.15.0.tgz'), 'TGZ');
+  const dir = scratchConsumer({ pinned: '1.14.1', installed: '1.14.1' });
+  const res = await u.runUpgrade(
+    { to: '1.15.0', from: join(src, 'bongos-core-1.15.0.tgz'), instance: dir, service: 'demo.service', healthUrl: 'http://x/healthz', versionUrl: 'http://x/v', rollbackOnFailure: false },
+    {
+      log: () => {}, err: () => {},
+      run: pinAwareRun([], { dir }),
+      materialize: () => ({ skills: 1 }), regenerateApiArtifacts: () => ({ ok: true, ran: [] }),
+      fetch: async (url) => (String(url) === 'http://x/v' ? { ok: false, status: 500 } : { ok: true, status: 200 }),
+      sleep: async () => {}, now: () => 0, healthBudgetMs: 0,
+    }
+  );
+  assert.equal(res.ok, false);
+  assert.equal(res.servedOk, false);
+});
+
+t('--no-health-check also opts out of the served-version read-back (one documented escape hatch)', async () => {
+  const src = mkdtempSync(join(tmpdir(), 'src-'));
+  writeFileSync(join(src, 'bongos-core-1.15.0.tgz'), 'TGZ');
+  const dir = scratchConsumer({ pinned: '1.14.1', installed: '1.14.1' });
+  let fetched = false;
+  const res = await u.runUpgrade(
+    { to: '1.15.0', from: join(src, 'bongos-core-1.15.0.tgz'), instance: dir, service: 'demo.service', healthCheck: false },
+    {
+      log: () => {}, err: () => {},
+      run: pinAwareRun([], { dir }),
+      materialize: () => ({ skills: 1 }), regenerateApiArtifacts: () => ({ ok: true, ran: [] }),
+      fetch: async () => { fetched = true; return { ok: true, status: 200 }; },
+    }
+  );
+  assert.equal(res.ok, true);
+  assert.equal(fetched, false, 'nothing is polled when the confirmation is explicitly waived');
+});
+
+t('the ROLLBACK restart escalates too — the restart after a failure is the one that matters', async () => {
+  const src = mkdtempSync(join(tmpdir(), 'src-'));
+  writeFileSync(join(src, 'bongos-core-1.15.0.tgz'), 'TGZ');
+  const dir = scratchConsumer({ pinned: '1.14.1', installed: '1.14.1' });
+  const calls = [], queries = [];
+  // The forward restart works; the health check then fails, and the rollback's restart is the
+  // one that needs sudo. Without deps reaching rollback's restartService there is no seam to
+  // assert this on, and the escalation that runs after a failure would go untested.
+  let restarts = 0;
+  const base = pinAwareRun(calls, { dir });
+  const run = (cmd, args = []) => {
+    if (cmd === 'systemctl' && args[0] === 'restart') {
+      restarts++;
+      calls.push(`systemctl ${args.join(' ')}`);
+      return { status: restarts === 1 ? 0 : 1 }; // forward ok, rollback needs escalation
+    }
+    if (cmd === 'sudo') { calls.push(`sudo ${args.join(' ')}`); return { status: 0 }; }
+    return base(cmd, args);
+  };
+  const res = await u.runUpgrade(
+    { to: '1.15.0', from: join(src, 'bongos-core-1.15.0.tgz'), instance: dir, service: 'demo.service', healthUrl: 'http://x/healthz' },
+    {
+      log: () => {}, err: () => {},
+      run, isRoot: false,
+      materialize: () => ({ skills: 1 }), regenerateApiArtifacts: () => ({ ok: true, ran: [] }),
+      databaseUrl: 'postgres://x/y', pg: recordingPg(queries),
+      fetch: async () => ({ ok: false, status: 503 }), // health fails forward AND after rollback
+      sleep: async () => {}, now: () => 0, healthBudgetMs: 0,
+    }
+  );
+  assert.equal(res.ok, false);
+  assert.equal(res.rolledBack, true);
+  assert.equal(res.rollback.restartOk, true, 'the rollback restart succeeded via sudo -n');
+  assert.ok(calls.includes('sudo -n systemctl restart demo.service'), 'the rollback path escalated');
+  assert.equal(u.readInstalledCoreVersion(dir), '1.14.1', 'the previous core is back on disk');
+});
+
+t('deriveVersionUrl: /version on the health URL origin, query + fragment dropped', () => {
+  assert.equal(u.deriveVersionUrl('http://127.0.0.1:3002/healthz'), 'http://127.0.0.1:3002/version');
+  assert.equal(u.deriveVersionUrl('https://demo.example.com/healthz?probe=1#x'), 'https://demo.example.com/version');
+  assert.equal(u.deriveVersionUrl('not a url'), null);
+});
+
+t('pollServedVersion: gives up rather than spinning when the clock never advances', async () => {
+  const r = await u.pollServedVersion('http://x/version', {
+    fetch: async () => ({ ok: true, status: 200, json: async () => ({}) }), // never carries a coreVersion
+    sleep: async () => {}, now: () => 0, versionBudgetMs: 1_000, versionMaxAttempts: 3,
+  });
+  assert.equal(r.ok, false);
+  assert.equal(r.attempts, 3, 'the attempt cap bounds a frozen clock');
+  assert.match(r.error, /no coreVersion/);
+});
+
 await Promise.all(pending);
 console.log(`\nupgrade: ${passed} passed, ${failed} failed`);
 process.exit(failed ? 1 : 0);