@bongos/core 1.19.399 → 1.19.400

package/modules/provisioning/provisioning.js

@@ -46,7 +46,11 @@ const PROJECT_TYPES = ['game', 'research', 'business', 'non-profit', 'not-sure']
 // per-project setting to an already-ACTIVE instance — patch its web.env settings
 // lines in place + restart, since the branding pack is memoized for the process
 // lifetime (task 1003139 built only the standup leg).
-const INTENT_ACTIONS = ['provision', 'teardown', 'oauth-manifest', 'domain-attach', 'settings-apply'];
+// 'restart' (task 1003238, migration provisioning_011): restart an already-ACTIVE
+// instance whose liveness probe says it is down — the settings-apply leg minus the
+// env patch. Its own action because no existing one fits: provision refuses
+// 'active' and settings-apply requires a value that actually changed.
+const INTENT_ACTIONS = ['provision', 'teardown', 'oauth-manifest', 'domain-attach', 'settings-apply', 'restart'];
 // Every state a teardown can be asked for — i.e. everything except the one
 // state where there is provably nothing left to release. DERIVED from
 // INSTANCE_STATES on purpose (task 1003042, walk finding N5): the old hand-kept
@@ -84,6 +88,19 @@ const SETTINGS_APPLY_FROM = new Set(['active']);
 // to it (the placeManifestCreds refusal) — so the route refuses the push HONESTLY
 // rather than queue one that never lands.
 const SETTINGS_APPLY_SHAPES = new Set(['co-tenant', 'standalone']);
+// The ONE state a restart is meaningful from (task 1003238). Liveness is an
+// OVERLAY, not a seventh state — an instance the heartbeat cannot reach is still
+// 'active', and 'active' is exactly where a restart applies. Every earlier state
+// has nothing running to restart and is already served by the provision path
+// (PROVISION_ENQUEUE_FROM covers 'requested'/'provisioning'/'error'/'torn_down');
+// 'tearing_down' is mid-reclaim. Kept as its own Set rather than reusing
+// SETTINGS_APPLY_FROM: each action declaring its own gate is this module's
+// convention, and the two answer different questions.
+const RESTART_FROM = new Set(['active']);
+// Same reachability rule as the settings push, for the same reason: a 'dedicated'
+// droplet is stood up by cloud-init and the runner has no post-hoc exec path to
+// it, so the route refuses HONESTLY rather than queue a restart that never lands.
+const RESTART_SHAPES = new Set(['co-tenant', 'standalone']);
 
 // Per-project settings (task 1003040, goal 1000058): the known keys + their
 // vocabularies, enforced at the API layer — the jsonb `settings` column
@@ -632,7 +649,10 @@ async function enqueueIntent(db, instanceId, action, requestedBy) {
 // (the state CHECK has no 'superseded'; last_error says why) so the runner never
 // stands it up. A PENDING settings push is superseded the same way (task 1003140):
 // a push into a project about to be torn down is dead work, and letting it hold the
-// delete hostage would repeat 1002691. A RUNNING intent is mid-flight and never yanked —
+// delete hostage would repeat 1002691. A pending 'restart' joins them for the same
+// reason (task 1003238) — and more sharply, since the owner reaches for Restart
+// precisely when a project looks broken, which is also when they give up and delete
+// it. A RUNNING intent is mid-flight and never yanked —
 // the caller gets { conflict: 'open_intent' } with the blocking intent so the route
 // can answer truthfully. Each statement is atomic and the index stays the arbiter:
 // any interleaving with the runner or a concurrent enqueue lands on one of these
@@ -642,7 +662,7 @@ async function enqueueTeardown(db, instanceId, requestedBy) {
     `UPDATE provisioning_intents
         SET state = 'error', last_error = 'superseded: teardown requested',
             updated_at = now(), resolved_at = now()
-      WHERE instance_id = $1 AND action IN ('provision', 'settings-apply') AND state = 'pending'
+      WHERE instance_id = $1 AND action IN ('provision', 'settings-apply', 'restart') AND state = 'pending'
       RETURNING id`,
     [instanceId]
   );
@@ -668,6 +688,23 @@ async function enqueueSettingsApply(db, instanceId, requestedBy) {
   return { intent, created: false, conflict: 'open_intent' };
 }
 
+// Queue a restart for an ACTIVE instance (task 1003238). Idempotent against its
+// own kind in EITHER open state, which is where it differs from the settings push:
+// that one must refuse to ride a RUNNING sibling because the runner may have read
+// the settings row before this write, so the in-flight run would not carry the
+// change. A restart carries no payload — a restart already running IS the thing
+// the owner asked for, so reporting "already under way" is the truth, not a
+// swallowed request. Anything else holding the single open slot is reported as
+// { conflict: 'open_intent' } so the route can name it. The state gate and the
+// shape gate are the ROUTE's job (RESTART_FROM / RESTART_SHAPES); this only
+// arbitrates the open slot, like enqueueTeardown.
+async function enqueueRestart(db, instanceId, requestedBy) {
+  const { intent, created } = await enqueueIntent(db, instanceId, 'restart', requestedBy);
+  if (created) return { intent, created: true };
+  if (intent.action === 'restart') return { intent, created: false };
+  return { intent, created: false, conflict: 'open_intent' };
+}
+
 // The instance's currently-open (pending|running) intent, if any.
 async function getOpenIntent(db, instanceId) {
   const { rows } = await db.query(
@@ -860,6 +897,7 @@ module.exports = {
   INSTANCE_STATES, HOSTING_SHAPES, ONBOARD_MODES, PROJECT_TYPES, INTENT_ACTIONS, TEARDOWN_FROM,
   settingsEnvLines, settingsEnvVars,
   PROVISION_ENQUEUE_FROM, DOMAIN_ATTACH_FROM, SETTINGS_APPLY_FROM, SETTINGS_APPLY_SHAPES,
+  RESTART_FROM, RESTART_SHAPES,
   TIER_MONTHLY_USD, CERT_ELIGIBLE_STATES,
   SETTINGS_VOCAB, DEFAULT_SETTINGS,
   // pure
@@ -873,7 +911,7 @@ module.exports = {
   // data access — verification overlay (task 1002696)
   listActiveInstancesForLiveness, recordLivenessProbe, recordDnsNote, clearErrorNote,
   createInstance, setInstanceStatus, updateInstanceDomain, updateInstanceSettings,
-  enqueueIntent, enqueueTeardown, enqueueSettingsApply, getOpenIntent, claimNextIntent, resolveIntent,
+  enqueueIntent, enqueueTeardown, enqueueSettingsApply, enqueueRestart, getOpenIntent, claimNextIntent, resolveIntent,
   recordEvent, instanceStateForSlug, costLedger,
   // data access — GitHub App Manifest attempts (task 2080)
   createOAuthManifest, getOAuthManifestByState, attachManifestCode,

package/modules/provisioning/routes/provisioning.js

@@ -795,6 +795,65 @@ module.exports = function provisioningRoutes() {
     }
   });
 
+  // POST /provisioning/instances/:id/restart — restart an ACTIVE project the
+  // heartbeat cannot reach (task 1003238). rank: any authenticated builder (own
+  // resource; archon may restart any — the override is audited via the actor).
+  //
+  // WHY it is its own route. The liveness overlay (task 1002696) can say a project
+  // is unreachable while its lifecycle status is still 'active' — and until now
+  // that reading came with no action the owner could take. Re-provision is not it:
+  // PROVISION_ENQUEUE_FROM deliberately excludes 'active' (task 1002691), so a
+  // Restart wired to that path would have refused on every press — the finding this
+  // route closes. The web tier only ENQUEUES; the control-plane runner performs the
+  // restart and the patient healthz verify (ADR 0111 §2).
+  //
+  // Every refusal names the action that IS available in that state rather than
+  // hiding behind a code, and no branch promises a duration — a restart's downtime
+  // is exactly what this route cannot know (the F16 rule, ADR 0190 §7).
+  router.post('/provisioning/instances/:id/restart', auth.requireBuilder, async (req, res) => {
+    const id = parseId(req, res); // (req, res) — parseId sends 400 bad_id + returns null on failure
+    if (!id) return;
+    try {
+      const inst = await provisioning.getInstanceById(pool, id);
+      if (instanceAccessDenied(inst, req.builder)) return res.fail('instance_not_found', 404);
+      // The archon override carries WHICH archon (the force-teardown convention).
+      const actor = String(inst.owner_builder_id) === String(req.builder.id) ? 'api:self' : `api:archon:${req.builder.id}`;
+      if (!provisioning.RESTART_FROM.has(inst.status)) {
+        const offline = inst.status === 'torn_down' || inst.status === 'tearing_down';
+        return res.fail('not_restartable', { status: 409, details: { status: inst.status }, message: offline
+          ? 'This project is offline, so there is nothing running to restart — stand it up again instead.'
+          : inst.status === 'error'
+            ? 'This project’s standup stopped partway, so there is nothing running to restart — stand it up again to run the standup afresh.'
+            : 'This project hasn’t finished standing up yet, so there is nothing to restart.' });
+      }
+      // A dedicated droplet is stood up by cloud-init and the runner has no post-hoc
+      // exec path to it — refuse honestly rather than queue a restart that never
+      // lands (the F16 lie), and name the command whoever runs that machine needs.
+      if (!provisioning.RESTART_SHAPES.has(inst.hosting_shape)) {
+        return res.fail('restart_unreachable', { status: 409, details: { hosting_shape: inst.hosting_shape },
+          message: `This project runs on a machine the platform doesn’t manage, so it can’t be restarted from here — whoever runs that machine has to run 'systemctl restart ${inst.slug}' there.` });
+      }
+      const r = await provisioning.enqueueRestart(pool, id, actor);
+      if (r.conflict) {
+        return res.fail('intent_conflict', { status: 409,
+          message: `A '${r.intent.action}' run is currently ${r.intent.state} for this project — the restart could not be queued; try again once that settles.`,
+          details: { action: r.intent.action, state: r.intent.state } });
+      }
+      if (r.created) {
+        await provisioning.recordEvent(pool, {
+          instanceId: id, ownerBuilderId: inst.owner_builder_id, event: 'restart',
+          detail: 'restart requested by owner', actor,
+        }).catch(() => {});
+      }
+      res.json({ ok: true, queued: r.created, action: 'restart', status: inst.status, message: r.created
+        ? 'Restart queued. Your project will stop and start again — this page says whether it answered afterwards.'
+        : 'A restart is already under way for this project.' });
+    } catch (err) {
+      log.error('[provisioning] POST /provisioning/instances/:id/restart', err);
+      res.fail('restart_failed', { status: 500, message: 'internal error' });
+    }
+  });
+
   // POST /provisioning/instances/:id/domain — give the caller's OWN instance its
   // address (task 1002692, gap 5: no path existed to add a domain after creation).
   // Validates + records the domain; for an already-ACTIVE instance it also enqueues

package/package-lock.json

@@ -1,12 +1,12 @@
 {
   "name": "@bongos/core",
-  "version": "1.19.399",
+  "version": "1.19.400",
   "lockfileVersion": 3,
   "requires": true,
   "packages": {
     "": {
       "name": "@bongos/core",
-      "version": "1.19.399",
+      "version": "1.19.400",
       "license": "AGPL-3.0-or-later",
       "dependencies": {
         "express": "^4.21.2",

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@bongos/core",
-  "version": "1.19.399",
+  "version": "1.19.400",
   "description": "Cloud Bongos — the AI-first build platform core (GDS + platform surfaces + module system), installed as a versioned dependency (ADR 0108).",
   "license": "AGPL-3.0-or-later",
   "main": "src/platform-server.js",

package/public-landing/projects.html

@@ -4023,6 +4023,17 @@ summary{min-height:24px;padding:3px 0;}
      both re-POST the slug (the server's PROVISION_ENQUEUE_FROM gate) */
   function canRetry(inst) { return inst.status === 'error' && !inst.open_intent; }
   function canRevive(inst) { return inst.status === 'torn_down' && !inst.open_intent; }
+  /* the restart door (task 1003238): client mirror of RESTART_FROM {active} +
+     RESTART_SHAPES (a dedicated droplet has no runner exec path), shown ONLY
+     where it has a reason to exist — a project the heartbeat cannot reach.
+     last_verified_up === false is the verdict; null means never probed and is
+     deliberately NOT the same claim (publicInstance keeps the two apart), so a
+     brand-new project never grows a Restart button it has no use for. A door on
+     every healthy card would only invite needless downtime. */
+  function canRestart(inst) {
+    return inst.status === 'active' && inst.last_verified_up === false &&
+      inst.hosting_shape !== 'dedicated' && !inst.open_intent;
+  }
   /* keep watching while the platform is doing something to this instance */
   function shouldWatch(inst) {
     if (inst.status === 'requested' || inst.status === 'provisioning' || inst.status === 'tearing_down') return true;
@@ -4053,7 +4064,8 @@ summary{min-height:24px;padding:3px 0;}
     revive: { names: ['Revive queued', 'Standing it up', 'Back live'], cls: '' },
     teardown: { names: ['Teardown queued', 'Winding down', 'Torn down'], cls: 'flowDown' },
     address: { names: ['Address queued', 'Wiring the address', 'Answering there'], cls: '' },
-    settings: { names: ['Change saved', 'Restarting to apply it', 'Applied'], cls: '' }
+    settings: { names: ['Change saved', 'Restarting to apply it', 'Applied'], cls: '' },
+    restart: { names: ['Restart queued', 'Restarting', 'Answering again'], cls: '' }
   };
   function stripMarkup(flow, stageNow, stagesDone, note, extraCls) {
     var f = FLOWS[flow];
@@ -4090,6 +4102,11 @@ summary{min-height:24px;padding:3px 0;}
       if (a === 'settings-apply') return inst.open_intent.state === 'running'
         ? stripMarkup('settings', 1, 1, 'Applying a settings change — the project restarts to pick it up.')
         : stripMarkup('settings', 0, 0, 'A settings change is queued — this card updates when it starts.');
+      /* task 1003238: the owner asked for a restart — say so, and never claim
+         it came back before a probe has said it did */
+      if (a === 'restart') return inst.open_intent.state === 'running'
+        ? stripMarkup('restart', 1, 1, 'Restarting the project — it goes down and comes back up.')
+        : stripMarkup('restart', 0, 0, 'A restart is queued — this card updates when it starts.');
       if (a === 'provision') return stripMarkup(s === 'torn_down' ? 'revive' : 'standup', 0, 0, (s === 'torn_down' ? 'A revive is queued' : 'A standup is queued') + ' — this card updates when it starts.');
       return stripMarkup('standup', 2, 2, 'A ' + esc(a) + ' run is queued — this card updates when it lands.');
     }
@@ -4176,7 +4193,17 @@ summary{min-height:24px;padding:3px 0;}
         /* the runner had no DNS hook (F16): the CAUSE beats the bare symptom —
            a successful A record flips last_verified_up and retires the note */
         if (inst.dns_note) return 'Its address is ' + link + '. ' + esc(inst.dns_note);
-        if (inst.last_verified_up === false) return 'Not answering at ' + link + ' right now. We keep checking every few minutes and update this on its own if it comes back.';
+        /* task 1003238: there IS an action now, so the sentence names it — the old
+           copy could only counsel patience because nothing else was on offer. It
+           names it through canRestart(), the SAME predicate the button renders on:
+           a dedicated-shape project (no runner exec path) or one with work already
+           queued gets no button, and pointing it at one would be the exact
+           misdirection ADR 0199 §8 forbids — a refusal must name the action that IS
+           available. Where there is no button the sentence stops at the full stop. */
+        if (inst.last_verified_up === false) {
+          return 'Not answering at ' + link + ' right now. We keep checking every few minutes and update this on its own if it comes back' +
+            (canRestart(inst) ? ' — or restart it yourself below.' : '.');
+        }
         if (inst.last_verified_up) return 'Its address is ' + link + ' — last answered ' + esc(relTime(verifiedAt(inst))) + '.';
         return 'Its address is ' + link + ' — confirming it answers.';
       }
@@ -4263,6 +4290,7 @@ summary{min-height:24px;padding:3px 0;}
     if (inst.status === 'active' && inst.domain) acts.unshift('<a class="btn small ghost" href="https://' + esc(inst.domain) + '" target="_blank" rel="noopener">Open site →</a>');
     if (canRetry(inst)) acts.push('<button type="button" class="btn small" data-retry="' + iid + '">Try again</button>');
     if (canRevive(inst)) acts.push('<button type="button" class="btn small" data-revive="' + iid + '">Bring it back</button>');
+    if (canRestart(inst)) acts.push('<button type="button" class="btn small" data-restart="' + iid + '">Restart it</button>');
     if (canTeardown(inst)) acts.push('<button type="button" class="btn small danger" data-teardown="' + iid + '">Tear it down</button>');
     if (acts.length) body += '<div class="actions" id="act-' + iid + '">' + acts.join('') + '</div>';
     body += attachFold(inst, iid);
@@ -4299,7 +4327,7 @@ summary{min-height:24px;padding:3px 0;}
     if (!el || !el.getAttribute) return null;
     if (el.tagName === 'SUMMARY') return 'details.attachFold summary';
     if (el.hasAttribute('data-teardown-yes') || el.hasAttribute('data-teardown-no')) return '[data-teardown]';
-    var keys = ['data-manage', 'data-attach', 'data-retry', 'data-revive', 'data-teardown'];
+    var keys = ['data-manage', 'data-attach', 'data-retry', 'data-revive', 'data-restart', 'data-teardown'];
     for (var i = 0; i < keys.length; i++) if (el.hasAttribute(keys[i])) return '[' + keys[i] + ']';
     return null;
   }
@@ -4539,6 +4567,7 @@ summary{min-height:24px;padding:3px 0;}
     if (inst.status === 'active' && inst.domain) acts.push('<a class="btn small ghost" href="https://' + esc(inst.domain) + '" target="_blank" rel="noopener">Open site →</a>');
     if (canRetry(inst)) acts.push('<button type="button" class="btn small" data-m-retry>Try again</button>');
     if (canRevive(inst)) acts.push('<button type="button" class="btn small" data-m-revive>Bring it back</button>');
+    if (canRestart(inst)) acts.push('<button type="button" class="btn small" data-m-restart>Restart it</button>');
     $('mActs').innerHTML = acts.join('');
     $('mActs').hidden = !acts.length;
   }
@@ -4575,11 +4604,33 @@ summary{min-height:24px;padding:3px 0;}
       box.textContent = 'Couldn’t reach the platform — check your connection and try again.';
     }).then(function () { loadManage(true, true); });
   }
+  /* Restart from the project's own page — the same POST the list card sends */
+  function mRestart() {
+    var inst = manageInst;
+    if (!inst) return;
+    var box = $('mMsg');
+    box.className = 'projMsg'; box.textContent = 'Asking the platform to restart it…';
+    $('mActs').querySelectorAll('button').forEach(function (b) { b.disabled = true; });
+    fetch(API + '/provisioning/instances/' + encodeURIComponent(inst.id) + '/restart', {
+      method: 'POST', credentials: 'same-origin'
+    }).then(function (res) {
+      if (res.status === 401) { box.className = 'projMsg err'; box.innerHTML = signInAgain(); return; }
+      return jsonOrEmpty(res).then(function (data) {
+        if (res.ok) { box.className = 'projMsg ok'; box.textContent = data.message || 'Restart queued.'; return; }
+        box.className = 'projMsg err';
+        box.textContent = restartFailure(data, res.status);
+      });
+    }).catch(function () {
+      box.className = 'projMsg err';
+      box.textContent = 'Couldn’t reach the platform — check your connection and try again.';
+    }).then(function () { loadManage(true, true); });
+  }
   $('mActs').addEventListener('click', function (ev) {
     var t = ev.target.closest && ev.target.closest('button');
     if (!t) return;
     if (t.hasAttribute('data-m-retry')) return mReProvision('retry');
     if (t.hasAttribute('data-m-revive')) return mReProvision('revive');
+    if (t.hasAttribute('data-m-restart')) return mRestart();
   });
 
   /* an IN-FLIGHT project is watched here at the card's cadence — its own copy
@@ -5315,6 +5366,39 @@ summary{min-height:24px;padding:3px 0;}
     });
   }
 
+  /* ── restart an unreachable project (task 1003238). Its own route, because
+     PROVISION_ENQUEUE_FROM excludes 'active' — re-POSTing the slug the way
+     Try again / Bring it back do would be refused on every press. Every
+     refusal is shown as the server worded it: the server owns the gate, and
+     a card that invents its own reason drifts from it. ── */
+
+  function restartFailure(data, status) {
+    var e = apiError(data, status);
+    if (e.code === 'rate_limited') return e.message || 'Too many requests right now — give it a moment and try again.';
+    return e.message || 'The restart could not be queued — try again in a moment.';
+  }
+
+  function doRestart(id) {
+    var inst = byId[id];
+    if (!inst) return;
+    quietActions(id);
+    msg(id, 'Asking…');
+    fetch(API + '/provisioning/instances/' + encodeURIComponent(id) + '/restart', {
+      method: 'POST', credentials: 'same-origin'
+    }).then(function (res) {
+      if (res.status === 401) { msg(id, signInAgain(), 'err'); return null; }
+      return jsonOrEmpty(res).then(function (data) {
+        if (res.ok) { msg(id, esc(data.message || 'Restart queued.'), 'ok'); return; }
+        msg(id, esc(restartFailure(data, res.status)), 'err');
+      });
+    }).catch(function () {
+      msg(id, 'Couldn’t reach the platform — check your connection and try again.', 'err');
+    }).then(function () {
+      if (byId[id]) renderCard(byId[id]);
+      watchOnce(id);
+    });
+  }
+
   /* ── domain attach: same truth-telling for every refusal (task 1002692) ── */
 
   function doAttach(id) {
@@ -5373,6 +5457,7 @@ summary{min-height:24px;padding:3px 0;}
     if (t.hasAttribute('data-attach')) return doAttach(t.getAttribute('data-attach'));
     if (t.hasAttribute('data-retry')) return reProvision(t.getAttribute('data-retry'), 'retry');
     if (t.hasAttribute('data-revive')) return reProvision(t.getAttribute('data-revive'), 'revive');
+    if (t.hasAttribute('data-restart')) return doRestart(t.getAttribute('data-restart'));
   });
 
   /* ── boot: who am I → my instances ────────────────────────────────────── */

package/scripts/gds/provision.js

@@ -768,6 +768,107 @@ async function settingsApplyInstance(inst, deps) {
   return { ok: true, healthy, consumed };
 }
 
+// ---------------------------------------------------------------------------
+// Restart an ALREADY-ACTIVE instance the heartbeat cannot reach (task 1003238 —
+// the honest owner action behind the card's "unreachable" line). This is
+// settingsApplyInstance minus the env patch: nothing about the instance's
+// configuration changes, so there is no file to read and no file to write — just
+// the restart and the same PATIENT healthz verify, whose result is recorded on
+// the liveness overlay so the card that sent the owner here updates itself.
+//
+// Why its own leg rather than a re-provision: PROVISION_ENQUEUE_FROM deliberately
+// excludes 'active' (task 1002691), and provisionInstance is a no-op on an active
+// instance by design — a full standup is the wrong tool for "turn it off and on
+// again". Why not settings-apply: that leg only runs behind a value that actually
+// CHANGED, which is exactly the case a stuck project does not have.
+//
+// Like the settings push, EVERY expected failure RETURNS { ok:false } instead of
+// throwing: the drain loop's catch flips the instance to 'error', and an instance
+// that is merely unreachable must not be relabelled broken by the very action
+// meant to recover it — that would drop it off the liveness sweep and leave a
+// re-provision as its only exit (ADR 0190 §2).
+// ---------------------------------------------------------------------------
+async function restartInstance(inst, deps) {
+  const { db, provisioning, exec, log, apply, privileged } = deps;
+  // Same box routing as settingsApplyInstance: a standalone runs control-plane-local.
+  const boxExec = inst.hosting_shape === 'standalone' ? (deps.controlExec || exec) : exec;
+  const sudoP = privileged ? 'sudo ' : '';
+  log(`restart ${inst.slug} (shape=${inst.hosting_shape}, status=${inst.status})`);
+
+  // A clean no-op (not an error) so a teardown that raced the restart never burns
+  // the intent's retry budget: every non-active state either has nothing running
+  // yet or is being reclaimed.
+  if (inst.status !== 'active') {
+    log(`  instance is '${inst.status}' — nothing running to restart`);
+    return { ok: true, noop: true };
+  }
+  // A dedicated droplet has no runner exec path (the placeManifestCreds refusal).
+  // The route refuses to enqueue for this shape too; this is the second gate.
+  if (inst.hosting_shape === 'dedicated') {
+    return { ok: false, error: `restart is not automated for a dedicated droplet — run 'systemctl restart ${inst.slug}' on that machine` };
+  }
+  if (!apply) {
+    boxExec(`${sudoP}systemctl restart ${inst.slug}`);
+    boxExec(healthzCmd(inst), { allowFail: true });
+    log(`  ✓ would restart ${inst.slug} and wait for it to answer (dry-run — nothing ran)`);
+    return { ok: true, healthy: null };
+  }
+  try {
+    log('  [service] restart');
+    boxExec(`${sudoP}systemctl restart ${inst.slug}`);
+  } catch (e) {
+    // ok:false, never a throw — the retry simply restarts again (idempotent).
+    return { ok: false, error: `could not restart ${inst.slug}: ${(e && e.message) || e}` };
+  }
+
+  // Verify it came back, with the manifest probe's patience (the F23 lesson). A
+  // false "down" is never cosmetic here: this instance is ALREADY flagged
+  // unreachable, and a hasty verdict would tell the owner their restart failed.
+  log('  [healthz] verify the instance answers after the restart');
+  const sleep = deps.sleep || ((ms) => new Promise((r) => setTimeout(r, ms)));
+  let healthy = false;
+  for (let i = 0; i < MANIFEST_VERIFY_TRIES; i++) {
+    const res = boxExec(healthzCmd(inst), { allowFail: true });
+    if (res && !res.softFailed) { healthy = true; break; }
+    if (i < MANIFEST_VERIFY_TRIES - 1) await sleep(MANIFEST_VERIFY_INTERVAL_MS);
+  }
+  // Same narrowness as the settings push: a loopback PASS proves the process, not
+  // the address, so only the domain-scoped sweep may claim "up" for an instance
+  // that HAS a domain. Silence after the whole budget is a verdict on any shape.
+  const probeIsVerdict = healthy === false || (healthy === true && !inst.domain);
+  if (probeIsVerdict && provisioning.recordLivenessProbe) await provisioning.recordLivenessProbe(db, inst.id, healthy).catch(() => {});
+  // A restart that worked retires any "we stopped trying" note left by earlier
+  // work (the F19 latch) — the note outliving its own recovery is the bug.
+  // DELIBERATELY gated on `healthy`, NOT on probeIsVerdict: the two answer
+  // different questions. probeIsVerdict asks "may this run claim the instance is
+  // UP?" — a loopback pass may not, for an instance whose address the sweep owns.
+  // The note asks "did work stop partway?", and a process that restarted and
+  // answered is that work finishing, whatever the public address does next.
+  // Holding the note until the sweep agrees would leave a stale "nothing more
+  // will happen" on the card of a project that just visibly recovered.
+  if (healthy && provisioning.clearErrorNote) await provisioning.clearErrorNote(db, inst.id).catch(() => {});
+  // Name WHAT was observed, not a verdict this probe cannot reach. healthzCmd is
+  // loopback (127.0.0.1:<port>) while the liveness sweep probes the PUBLIC
+  // https://<domain>/healthz — so an instance can answer here while DNS, Caddy or
+  // TLS is the thing actually broken. "Answering again" on a domained instance
+  // would contradict the card still reading "unreachable"; the same asymmetry is
+  // why probeIsVerdict above declines to write an up-verdict for that case.
+  const outcome = !healthy ? 'restarted, but it is not answering on the box — something there needs a look'
+    : inst.domain ? 'restarted and answering on the box; whether its public address answers is for the next liveness sweep to say'
+    : 'restarted and answering again';
+  await provisioning.recordEvent(db, {
+    instanceId: inst.id, ownerBuilderId: inst.owner_builder_id, event: 'restarted',
+    detail: `restart requested — ${outcome}`,
+    actor: 'runner',
+  }).catch(() => {});
+  log(healthy ? `  ✓ ${inst.slug} ${outcome}` : `  ⚠ ${inst.slug} ${outcome}`);
+  // ok:true either way: the restart RAN. A project that stays down is a truthful
+  // observation for the card, not a failed intent to retry MAX_INTENT_ATTEMPTS
+  // times — restarting a broken instance four more times fixes nothing and would
+  // end by stamping an error_note over the owner's own deliberate action.
+  return { ok: true, healthy };
+}
+
 // ---------------------------------------------------------------------------
 // Teardown one instance (reverse the runbook; idempotent).
 // ---------------------------------------------------------------------------
@@ -1049,6 +1150,7 @@ async function cmdRunIntents(deps) {
       else if (intent.action === 'teardown') result = await teardownInstance(inst, deps);
       else if (intent.action === 'domain-attach') result = await domainAttachInstance(inst, deps);
       else if (intent.action === 'settings-apply') result = await settingsApplyInstance(inst, deps);
+      else if (intent.action === 'restart') result = await restartInstance(inst, deps);
       else result = { ok: false, error: `unknown intent action '${intent.action}' — refusing to guess` };
       if (result && result.ok === false) {
         if (intent.attempts >= MAX_INTENT_ATTEMPTS) {
@@ -1238,6 +1340,8 @@ module.exports = {
   domainAttachInstance, cmdRunIntents, cmdReconcile, cmdStatus,
   // live settings push (task 1003140) + the in-place env patch + read-back it rides on
   settingsApplyInstance, upsertEnvVars, settingsEnvVarsFor, settingsConsumed, instanceManifestCmd,
+  // restart an unreachable ACTIVE instance (task 1003238) — the settings leg minus the env patch
+  restartInstance,
   // GitHub App Manifest flow (task 2080, ADR 0133)
   convertManifestCode, placeManifestCreds, drainOAuthManifests, MANIFEST_UA,
   // cross-site identity federation (ADR 0141 §4, task #1002109)

package/src/module-api.js

@@ -1238,8 +1238,10 @@ const { validateOrRespond, LIMITS, parseId, asyncHandler, corsPublicGet, parsePa
 //         landed since 1.19.397 with no explicit bump. run 33199474439. (task 1002620)
 //   1.19.399 — CI auto-patch (publish-on-merge, ADR 0161): carrier for merges
 //         landed since 1.19.398 with no explicit bump. run 33201243958. (task 1002620)
+//   1.19.400 — CI auto-patch (publish-on-merge, ADR 0161): carrier for merges
+//         landed since 1.19.399 with no explicit bump. run 33201979306. (task 1002620)
 // ---------------------------------------------------------------------------
-const CORE_VERSION = '1.19.399'; // CI auto-patch carrier (ADR 0161); see changelog above
+const CORE_VERSION = '1.19.400'; // CI auto-patch carrier (ADR 0161); see changelog above
 
 // A namespaced logger so a module's log lines are attributable + consistent.
 // Usage: const log = api.logger('dev-box'); log.info('mounted');

package/tests/projects_hub_pre_uat.mjs

@@ -54,6 +54,46 @@ test('addressRow branches by state before it trusts the domain — torn down and
   assert.match(body, /We keep checking every few minutes/, 'an unreachable project says what the platform is doing');
 });
 
+/* ── the restart door for an unreachable project (task 1003238) ── */
+test('the restart door appears ONLY where it has a reason to exist, and mirrors the server gate', () => {
+  const body = fn('canRestart');
+  assert.match(body, /inst\.status === 'active'/, "RESTART_FROM is {active}");
+  assert.match(body, /inst\.last_verified_up === false/, 'the probe verdict is the trigger — a healthy card grows no Restart button');
+  assert.doesNotMatch(body, /last_verified_up === null|!inst\.last_verified_up/,
+    'null (never probed) is deliberately NOT the same claim as false — publicInstance keeps the two apart');
+  assert.match(body, /inst\.hosting_shape !== 'dedicated'/, 'RESTART_SHAPES — the runner has no exec path to a droplet');
+  assert.match(body, /!inst\.open_intent/, 'the strip tells the story while work is in flight');
+});
+
+test('the restart door is offered on BOTH surfaces, and the address row points at it', () => {
+  assert.match(fn('renderCard'), /canRestart\(inst\)[\s\S]*?data-restart=/, 'the list card');
+  assert.match(fn('renderManage'), /canRestart\(inst\)[\s\S]*?data-m-restart/, 'the project’s own page — the two never disagree');
+  // the door and the sentence that sends the owner to it must ship together —
+  // and the sentence must be gated by the SAME predicate, or a project with no
+  // button (dedicated shape, or work already queued) is told to scroll to one
+  const addr = fn('addressRow');
+  assert.match(addr, /or restart it yourself below/, 'the unreachable line names the action now that one exists');
+  assert.match(addr, /canRestart\(inst\) \? ' — or restart it yourself below\.' : '\.'/,
+    'the pointer is conditioned on canRestart(), not on last_verified_up alone (ADR 0199 §8)');
+});
+
+test('a queued restart has its own strip — never the generic "a <action> run is queued" fallthrough', () => {
+  assert.match(SCRIPT, /restart: \{ names: \['Restart queued', 'Restarting', 'Answering again'\]/, 'the flow exists');
+  const body = fn('stripFor');
+  assert.match(body, /a === 'restart'/, 'stripFor branches on it');
+  assert.match(body, /Restarting the project — it goes down and comes back up\./, 'the running state says what is happening');
+  // the third stage is "Answering again" — it must not be lit by the strip alone,
+  // exactly like every other flow here (the probe earns that claim, not the queue)
+  assert.doesNotMatch(body, /stripMarkup\('restart', 2, 2/, 'the queue never lights the final stage');
+});
+
+test('the restart sender posts to the restart ROUTE — re-POSTing the slug would be refused on every press', () => {
+  const body = fn('doRestart');
+  assert.match(body, /\/restart'/, 'its own endpoint');
+  assert.doesNotMatch(body, /provisioning\/instances',/, 'NOT the create/re-provision POST — PROVISION_ENQUEUE_FROM excludes active');
+  assert.match(body, /quietActions\(id\)/, 'the row goes quiet while it is in flight');
+  assert.match(fn('mRestart'), /\/restart'/, 'and the manage page sends the same request');
+});
 /* ── the moment a standup lands (task 1003243) ── */
 test('the landed moment is earned by a VERIFIED probe inside a window, not invented state', () => {
   const body = fn('justLive');