@bongos/core 1.19.399 → 1.19.400

package/tests/provision_restart.mjs

@@ -0,0 +1,242 @@
+// tests/provision_restart.mjs
+//
+// The RUNNER half of the restart path for an unreachable ACTIVE project (task
+// 1003238, goal 1000058): scripts/gds/provision.js restartInstance and the
+// run-intents dispatch that reaches it. Hermetic — exec / db are recording
+// fakes; no shell, no DB, no network (the tests/provision_settings_apply.mjs
+// pattern).
+//
+// The load-bearing assertions:
+//   • it is the settings-apply leg MINUS the env patch: one systemctl restart,
+//     then the patient healthz verify — and it NEVER reads or writes web.env,
+//     because nothing about the instance's configuration is changing
+//   • a restart that THROWS returns ok:false so the drain loop re-pends it, and
+//     an instance that is merely unreachable is never relabelled 'error' by the
+//     very action meant to recover it (ADR 0190 §2)
+//   • the healthz budget is the PATIENT one — a false "down" here would tell the
+//     owner their restart failed
+//   • the probe verdict keeps the settings-apply narrowness: a loopback PASS on a
+//     DOMAINED instance is not an up-verdict (the sweep probes the public
+//     address), and the event text says which one was actually observed
+//   • a project that stays down is ok:true — a truthful observation for the card,
+//     not a failure to retry four more times
+//   • the state gate is a clean no-op; dedicated is refused with the command
+//   • cmdRunIntents routes 'restart' to this leg
+//
+// Run:  node tests/provision_restart.mjs
+
+import { strict as assert } from 'node:assert';
+import { createRequire } from 'node:module';
+
+const require = createRequire(import.meta.url);
+const P = require('../scripts/gds/provision.js');
+const { MANIFEST_VERIFY_TRIES } = require('../scripts/gds/provision-config.js');
+
+let passed = 0, failed = 0;
+async function ta(name, fn) {
+  try { await fn(); passed++; console.log(`  PASS  ${name}`); }
+  catch (e) { failed++; console.log(`  FAIL  ${name}\n        ${e.message}`); }
+}
+
+function recorder({ apply = true, healthzFails = false, privileged = false, restartThrows = false } = {}) {
+  const cmds = [], writes = [], events = [], probes = [], cleared = [], logs = [];
+  const exec = (cmd) => {
+    cmds.push(cmd);
+    if (/healthz/.test(cmd)) return healthzFails ? { ok: true, stdout: '', softFailed: true } : { ok: true, stdout: '{"ok":true}' };
+    if (/systemctl restart/.test(cmd) && restartThrows && apply) throw new Error(`Command failed: ${cmd}`);
+    return { ok: true, stdout: '' };
+  };
+  const writer = (via) => (p, body, opts) => { writes.push({ via, path: p, body, opts }); return { ok: true }; };
+  const deps = {
+    apply, privileged, log: (l) => logs.push(String(l)), sleep: async () => {},
+    db: { query: async () => ({ rows: [] }) },
+    provisioning: {
+      recordEvent: async (_db, e) => { events.push(e); },
+      recordLivenessProbe: async (_db, id, up) => { probes.push({ id, up }); },
+      clearErrorNote: async (_db, id) => { cleared.push(id); },
+    },
+    exec,
+    writeFile: writer('box'), controlWriteFile: writer('control'),
+    doApi: null,
+  };
+  return { deps, cmds, writes, events, probes, cleared, logs };
+}
+
+const activeStandalone = {
+  id: 18, slug: 'spike', hosting_shape: 'standalone', status: 'active', owner_builder_id: 7,
+  port: 3010, db_name: 'spike', domain: 'spike.cloudbongos.com', target_ref: 'jaxri/spike-harbor',
+};
+
+console.log('restartInstance — the leg:');
+
+await ta('--apply, active standalone: ONE systemctl restart, then healthz — and web.env is never read or written', async () => {
+  const { deps, cmds, writes } = recorder();
+  const r = await P.restartInstance(activeStandalone, deps);
+  assert.equal(r.ok, true, JSON.stringify(r));
+  assert.equal(writes.length, 0, 'nothing about the configuration changes, so nothing is written');
+  assert.ok(!cmds.some((c) => /\bcat /.test(c)), 'no web.env read — this is not the settings push');
+  const restarts = cmds.filter((c) => /systemctl restart spike/.test(c));
+  assert.equal(restarts.length, 1, 'exactly one restart');
+  const idx = (re) => cmds.findIndex((c) => re.test(c));
+  assert.ok(idx(/systemctl restart spike/) < idx(/healthz/), 'healthz comes after the restart');
+  assert.ok(!cmds.some((c) => /systemctl (stop|disable)|caddy|box-dns|dropdb/.test(c)), 'no teardown / DNS / caddy legs');
+});
+
+await ta('the healthz verify is the PATIENT budget — a false "down" would tell the owner their restart failed', async () => {
+  const { deps, cmds } = recorder({ healthzFails: true });
+  const r = await P.restartInstance(activeStandalone, deps);
+  assert.equal(r.ok, true);
+  assert.equal(r.healthy, false);
+  assert.equal(cmds.filter((c) => /healthz/.test(c)).length, MANIFEST_VERIFY_TRIES, 'the whole budget is spent before the verdict');
+});
+
+await ta('a project that stays down is ok:true — a truthful observation for the card, not a failure to retry four more times', async () => {
+  const { deps, events, probes } = recorder({ healthzFails: true });
+  const r = await P.restartInstance(activeStandalone, deps);
+  assert.equal(r.ok, true, 'restarting a broken instance again fixes nothing and would end by stamping an error_note over the owner’s own action');
+  assert.equal(r.healthy, false);
+  assert.deepEqual(probes, [{ id: 18, up: false }], 'silence after the whole budget IS a verdict');
+  assert.equal(events[0].event, 'restarted');
+  assert.match(events[0].detail, /not answering on the box/);
+});
+
+await ta('a restart that THROWS returns ok:false — the drain loop re-pends it and the instance keeps its status', async () => {
+  const { deps, events, probes } = recorder({ restartThrows: true });
+  const r = await P.restartInstance(activeStandalone, deps);
+  assert.equal(r.ok, false);
+  assert.match(r.error, /could not restart spike/);
+  assert.equal(events.length, 0, 'nothing claimed about an attempt that did not run');
+  assert.equal(probes.length, 0, 'and no liveness verdict invented from it');
+});
+
+await ta('a loopback PASS on a DOMAINED instance is NOT an up-verdict, and the event says which probe actually answered', async () => {
+  const { deps, events, probes } = recorder();
+  const r = await P.restartInstance(activeStandalone, deps);
+  assert.equal(r.healthy, true);
+  assert.equal(probes.length, 0, 'the sweep probes the PUBLIC address — loopback proves the process, not the address');
+  assert.match(events[0].detail, /answering on the box/, 'names what was observed');
+  assert.match(events[0].detail, /public address/, 'and is explicit that the public verdict is still outstanding');
+  assert.doesNotMatch(events[0].detail, /answering again/, 'never a claim the card would contradict with "unreachable"');
+});
+
+await ta('with NO domain, loopback IS the whole claim — the up-verdict is recorded and the error latch released', async () => {
+  const { deps, events, probes, cleared } = recorder();
+  const r = await P.restartInstance({ ...activeStandalone, domain: null }, deps);
+  assert.equal(r.healthy, true);
+  assert.deepEqual(probes, [{ id: 18, up: true }]);
+  assert.deepEqual(cleared, [18], 'a restart that worked retires a stale "we stopped trying" note');
+  assert.match(events[0].detail, /answering again/);
+});
+
+await ta('a DOMAINED instance that answers on loopback still releases the latch — the note asks whether work stopped, not whether the address answers', async () => {
+  const { deps, probes, cleared } = recorder();
+  const r = await P.restartInstance(activeStandalone, deps);
+  assert.equal(r.healthy, true);
+  assert.equal(probes.length, 0, 'no up-VERDICT is claimed — that one belongs to the public sweep');
+  assert.deepEqual(cleared, [18],
+    'but the F19 latch clears: holding it until the sweep agrees would leave a stale "nothing more will happen" on a project that just recovered');
+});
+
+await ta('a failed restart does NOT release the error latch', async () => {
+  const { deps, cleared } = recorder({ healthzFails: true });
+  await P.restartInstance({ ...activeStandalone, domain: null }, deps);
+  assert.equal(cleared.length, 0, 'the note outliving its own recovery is the bug — but so is clearing it without one');
+});
+
+await ta('every non-active state is a clean no-op, not an error — a raced teardown never burns the retry budget', async () => {
+  for (const status of ['requested', 'provisioning', 'tearing_down', 'torn_down', 'error']) {
+    const { deps, cmds } = recorder();
+    const r = await P.restartInstance({ ...activeStandalone, status }, deps);
+    assert.equal(r.ok, true, status);
+    assert.equal(r.noop, true, status);
+    assert.equal(cmds.length, 0, `${status} ran no command`);
+  }
+});
+
+await ta('a dedicated droplet is refused with the exact command, and nothing is run', async () => {
+  const { deps, cmds } = recorder();
+  const r = await P.restartInstance({ ...activeStandalone, hosting_shape: 'dedicated' }, deps);
+  assert.equal(r.ok, false);
+  assert.match(r.error, /systemctl restart spike/);
+  assert.equal(cmds.length, 0);
+});
+
+await ta('dry-run runs no real work and observes nothing', async () => {
+  const { deps, writes } = recorder({ apply: false });
+  const r = await P.restartInstance(activeStandalone, deps);
+  assert.equal(r.ok, true);
+  assert.equal(r.healthy, null, 'nothing ran, nothing observed');
+  assert.equal(writes.length, 0);
+});
+
+await ta('a co-tenant restarts on the co-hosting box; a standalone on the control plane', async () => {
+  const seen = {};
+  for (const shape of ['co-tenant', 'standalone']) {
+    const box = [], control = [];
+    const { deps } = recorder();
+    deps.exec = (cmd) => { box.push(cmd); return /healthz/.test(cmd) ? { ok: true, stdout: '{}' } : { ok: true, stdout: '' }; };
+    deps.controlExec = (cmd) => { control.push(cmd); return /healthz/.test(cmd) ? { ok: true, stdout: '{}' } : { ok: true, stdout: '' }; };
+    await P.restartInstance({ ...activeStandalone, hosting_shape: shape }, deps);
+    seen[shape] = { box: box.length, control: control.length };
+  }
+  assert.ok(seen['co-tenant'].box > 0 && seen['co-tenant'].control === 0, 'a co-tenant runs on the box');
+  assert.ok(seen.standalone.control > 0 && seen.standalone.box === 0, 'a standalone ran control-plane-local (task 2074) — restart it THERE');
+});
+
+await ta('privileged mode escalates the restart', async () => {
+  const { deps, cmds } = recorder({ privileged: true });
+  await P.restartInstance(activeStandalone, deps);
+  assert.ok(cmds.some((c) => /^sudo systemctl restart spike$/.test(c)), cmds.join(' | '));
+});
+
+console.log('\ncmdRunIntents — the dispatch:');
+
+await ta("cmdRunIntents (--apply) routes 'restart' to restartInstance and resolves it done", async () => {
+  let claims = 0; const resolved = []; const statusWrites = []; const cmds = [];
+  const deps = {
+    apply: true, log: () => {}, sleep: async () => {},
+    db: { query: async () => ({ rows: [] }) },
+    provisioning: {
+      claimNextIntent: async () => (claims++ === 0 ? { id: 12, instance_id: 18, action: 'restart', attempts: 1 } : null),
+      getInstanceById: async () => ({ ...activeStandalone }),
+      resolveIntent: async (_db, id, state, err) => { resolved.push({ id, state, err }); },
+      setInstanceStatus: async (_db, _id, s, patch) => { statusWrites.push({ s, patch }); },
+      recordEvent: async () => {}, recordLivenessProbe: async () => {}, clearErrorNote: async () => {},
+      claimNextOAuthManifest: async () => null, resolveOAuthManifest: async () => {},
+    },
+    exec: (cmd) => { cmds.push(cmd); return /healthz/.test(cmd) ? { ok: true, stdout: '{}' } : { ok: true, stdout: '' }; },
+    writeFile: () => ({ ok: true }), doApi: null,
+  };
+  const r = await P.cmdRunIntents(deps);
+  assert.equal(r.errors, 0, JSON.stringify(r));
+  assert.equal(resolved[0].state, 'done');
+  assert.ok(cmds.some((c) => /systemctl restart spike/.test(c)), 'it really reached the restart leg');
+  assert.equal(statusWrites.length, 0, 'a restart never touches lifecycle status');
+});
+
+await ta('cmdRunIntents (--apply): a restart that throws re-pends and never marks a healthy ACTIVE instance broken', async () => {
+  let claims = 0; const resolved = []; const statusWrites = []; const updates = [];
+  const deps = {
+    apply: true, log: () => {}, sleep: async () => {},
+    db: { query: async (sql, params) => { if (/UPDATE provisioning_intents/.test(sql)) updates.push({ sql, params }); return { rows: [] }; } },
+    provisioning: {
+      claimNextIntent: async () => (claims++ === 0 ? { id: 12, instance_id: 18, action: 'restart', attempts: 1 } : null),
+      getInstanceById: async () => ({ ...activeStandalone }),
+      resolveIntent: async (_db, id, state, err) => { resolved.push({ id, state, err }); },
+      setInstanceStatus: async (_db, _id, s, patch) => { statusWrites.push({ s, patch }); },
+      recordEvent: async () => {}, recordLivenessProbe: async () => {}, clearErrorNote: async () => {},
+      claimNextOAuthManifest: async () => null, resolveOAuthManifest: async () => {},
+    },
+    exec: (cmd) => { if (/systemctl restart/.test(cmd)) throw new Error('Command failed: sudo systemctl restart spike'); return { ok: true, stdout: '' }; },
+    writeFile: () => ({ ok: true }), doApi: null,
+  };
+  const r = await P.cmdRunIntents(deps);
+  assert.equal(r.errors, 1);
+  assert.equal(resolved.length, 0, 'not resolved — it re-pends for a later tick');
+  assert.match(updates[0].sql, /state='pending'/);
+  assert.match(updates[0].params[1], /could not restart spike/, 'the reason rides last_error');
+  assert.equal(statusWrites.length, 0, 'an unreachable instance is not relabelled broken by the action meant to recover it');
+});
+
+console.log(`\nprovision_restart: ${passed} passed, ${failed} failed`);
+process.exit(failed ? 1 : 0);

package/tests/provisioning_restart.mjs

@@ -0,0 +1,285 @@
+// tests/provisioning_restart.mjs
+//
+// The RESTART path for an unreachable ACTIVE project (task 1003238, goal
+// 1000058) — web-tier half. The liveness overlay (task 1002696) can report a
+// project unreachable while its lifecycle status is still 'active', and until
+// this route existed that reading came with no action the owner could take:
+// PROVISION_ENQUEUE_FROM deliberately excludes 'active' (task 1002691), so a
+// Restart wired to the re-provision path would have refused on every press.
+//
+// Covers (the task's done-when, web-tier side):
+//   • the STATE GATE — RESTART_FROM is exactly {active}, and the premise it rests
+//     on (PROVISION_ENQUEUE_FROM excludes 'active') is asserted, not assumed
+//   • the SHAPE REFUSAL — a dedicated droplet is refused honestly and told the
+//     command to run, never queued a restart that cannot land (the F16 rule)
+//   • the open-slot arbitration — a foreign intent → 409 intent_conflict naming
+//     the real blocker; an already-open RESTART is idempotent in either state,
+//     which is where it differs from the settings push
+//   • a pending restart is superseded by a teardown (the 1002691 class)
+//   • every refusal names the action that IS available in that state, and no
+//     branch promises a duration
+//   • INTENT_ACTIONS <-> migration provisioning_011 lockstep (the strict
+//     set-equality check lives with the NEWEST migration)
+//   • the archon override is own-scoped and audited (api:archon:<id>)
+//
+// DB-free: data-access fns are stubbed on the cached module object (the
+// provisioning_settings_apply.mjs pattern). No network beyond localhost.
+//
+// Run:  node tests/provisioning_restart.mjs
+
+import { strict as assert } from 'node:assert';
+import { createRequire } from 'node:module';
+import { readFileSync } from 'node:fs';
+import http from 'node:http';
+
+const require = createRequire(import.meta.url);
+const provisioning = require('../modules/provisioning/provisioning.js');
+
+let passed = 0, failed = 0;
+function t(name, fn) {
+  try { fn(); passed++; console.log(`  PASS  ${name}`); }
+  catch (e) { failed++; console.log(`  FAIL  ${name}\n        ${e.message}`); }
+}
+async function ta(name, fn) {
+  try { await fn(); passed++; console.log(`  PASS  ${name}`); }
+  catch (e) { failed++; console.log(`  FAIL  ${name}\n        ${e.message}`); }
+}
+
+// No wall-clock or duration promise anywhere a message is rendered (F16). A
+// restart's downtime is precisely what the web tier cannot know.
+const PROMISE = /shortly|briefly|minute|second|soon\b/i;
+
+console.log('the module slice — constants, the migration, the enqueue classifier:');
+
+t("INTENT_ACTIONS includes 'restart', in lockstep with the provisioning_011 CHECK", () => {
+  assert.ok(provisioning.INTENT_ACTIONS.includes('restart'));
+  const sql = readFileSync(new URL('../modules/provisioning/migrations/provisioning_011_restart.sql', import.meta.url), 'utf8');
+  const m = /CHECK \(action IN \(([^)]*)\)\)/.exec(sql);
+  assert.ok(m, 'the migration widens the action CHECK');
+  const inSql = m[1].split(',').map((s) => s.trim().replace(/^'|'$/g, '')).sort();
+  // The NEWEST migration owns the strict set equality — the CHECK a live DB
+  // ends up with is the last one applied, so this is the assertion that would
+  // catch an action added in code but never admitted by the schema.
+  assert.deepEqual(inSql, [...provisioning.INTENT_ACTIONS].sort(), 'the CHECK and INTENT_ACTIONS carry the same set');
+  assert.match(sql, /INSERT INTO schema_migrations \(version\) VALUES \('provisioning_011_restart'\)/, 'self-registers');
+});
+
+t('RESTART_FROM is exactly {active} — every other state is already served by the provision path', () => {
+  assert.deepEqual([...provisioning.RESTART_FROM], ['active']);
+  // The whole reason this route exists: the re-provision door cannot cover it.
+  assert.ok(!provisioning.PROVISION_ENQUEUE_FROM.has('active'), 'the premise the gate rests on');
+  for (const s of ['requested', 'provisioning', 'error', 'torn_down']) {
+    assert.ok(provisioning.PROVISION_ENQUEUE_FROM.has(s), `${s} has the re-provision door instead`);
+  }
+});
+
+t('RESTART_SHAPES is the runner-reachable pair — dedicated is out, and the set is a subset of HOSTING_SHAPES', () => {
+  assert.deepEqual([...provisioning.RESTART_SHAPES].sort(), ['co-tenant', 'standalone']);
+  assert.ok(!provisioning.RESTART_SHAPES.has('dedicated'));
+  for (const s of provisioning.RESTART_SHAPES) assert.ok(provisioning.HOSTING_SHAPES.includes(s), `${s} is a real shape`);
+});
+
+await ta('enqueueRestart: a free slot creates the intent', async () => {
+  const db = { query: async (_sql, params) => ({ rows: [{ id: 9, action: 'restart', state: 'pending', instance_id: params[0] }] }) };
+  const r = await provisioning.enqueueRestart(db, 18, 'api:self');
+  assert.equal(r.created, true);
+  assert.equal(r.intent.action, 'restart');
+});
+
+await ta('enqueueRestart: an already-open restart is idempotent in EITHER state — a restart carries no payload, so one already running IS the ask', async () => {
+  for (const state of ['pending', 'running']) {
+    const db = { query: async (sql) => {
+      if (/INSERT/.test(sql)) { const e = new Error('dup'); e.code = '23505'; throw e; }
+      return { rows: [{ id: 5, action: 'restart', state }] };
+    } };
+    const r = await provisioning.enqueueRestart(db, 18, 'api:self');
+    assert.equal(r.created, false, state);
+    assert.equal(r.conflict, undefined, `a running restart is not a conflict (unlike a settings push, which may have read the row already) — ${state}`);
+  }
+});
+
+await ta('enqueueRestart: any OTHER action holding the slot → conflict, naming the real blocker', async () => {
+  for (const open of [{ action: 'teardown', state: 'pending' }, { action: 'domain-attach', state: 'running' }, { action: 'settings-apply', state: 'pending' }]) {
+    const db = { query: async (sql) => {
+      if (/INSERT/.test(sql)) { const e = new Error('dup'); e.code = '23505'; throw e; }
+      return { rows: [{ id: 5, ...open }] };
+    } };
+    const r = await provisioning.enqueueRestart(db, 18, 'api:self');
+    assert.equal(r.created, false, JSON.stringify(open));
+    assert.equal(r.conflict, 'open_intent', JSON.stringify(open));
+    assert.equal(r.intent.action, open.action, 'names the real blocker');
+  }
+});
+
+await ta('enqueueTeardown supersedes a PENDING restart — the owner reaches for Restart when a project looks broken, which is also when they give up on it', async () => {
+  const calls = [];
+  const db = { query: async (sql, params) => {
+    calls.push(sql);
+    if (/UPDATE provisioning_intents/.test(sql)) return { rows: [{ id: 5 }] };
+    return { rows: [{ id: 6, action: 'teardown', state: 'pending', instance_id: params[0] }] };
+  } };
+  const r = await provisioning.enqueueTeardown(db, 18, 'api:self');
+  assert.equal(r.superseded, true);
+  assert.match(calls[0], /'restart'/, 'restart joined the supersedable list');
+  assert.match(calls[0], /state = 'pending'/, 'pending only — a RUNNING intent is never yanked');
+});
+
+// ── the route, over a real mounted router (stubbed data access) ─────────────────
+let instance = null;
+let enqueueCalls = [];
+let enqueueResult = null;   // what enqueueRestart reports (null → created:true)
+let events = [];
+provisioning.getInstanceById = async (_db, id) => (instance ? { ...instance, id } : null);
+provisioning.enqueueRestart = async (_db, id, requestedBy) => {
+  enqueueCalls.push({ id, requestedBy });
+  return enqueueResult || { intent: { id: 9, action: 'restart', state: 'pending' }, created: true };
+};
+provisioning.recordEvent = async (_db, e) => { events.push(e); };
+provisioning.getOpenIntent = async () => null;
+provisioning.getLatestOAuthManifestForInstance = async () => null;
+
+const api = require('../src/module-api.js');
+let builder = { id: 7, github_login: 'owner', rank: 'thetes' };
+api.requireBuilder = (req, _res, next) => { req.builder = builder; next(); };
+api.requirePermission = () => (_req, _res, next) => next();
+api.branding = () => ({ domains: { publicOrigin: 'https://cb.test' }, identity: { productName: 'Cloud Bongos' } });
+api.logger = () => ({ error() {}, warn() {}, info() {}, debug() {} });
+
+const express = require('express');
+const provisioningRoutes = require('../modules/provisioning/routes/provisioning.js');
+const app = express();
+app.use(express.json());
+app.use((_req, res, next) => {
+  res.fail = (code, opts, details) => {
+    const status = typeof opts === 'number' ? opts : (opts && opts.status) || 400;
+    res.status(status).json({ error: { code, ...(typeof opts === 'object' ? opts : {}), ...(details || {}) } });
+  };
+  next();
+});
+app.use('/api/bongos', provisioningRoutes());
+
+const server = app.listen(0);
+await new Promise((r) => server.once('listening', r));
+const base = `http://127.0.0.1:${server.address().port}/api/bongos`;
+
+function post(path) {
+  return new Promise((resolve, reject) => {
+    const req = http.request(`${base}${path}`, { method: 'POST' }, (res) => {
+      let d = '';
+      res.on('data', (c) => { d += c; });
+      res.on('end', () => { let j = null; try { j = JSON.parse(d); } catch { /* non-JSON */ } resolve({ status: res.statusCode, body: j, raw: d }); });
+    });
+    req.on('error', reject);
+    req.end();
+  });
+}
+
+const ACTIVE = { owner_builder_id: 7, slug: 'spike', status: 'active', hosting_shape: 'standalone', domain: 'spike.cloudbongos.com', settings: {} };
+function reset(patch = {}) {
+  instance = { ...ACTIVE, ...patch };
+  enqueueCalls = []; enqueueResult = null; events = [];
+  builder = { id: 7, github_login: 'owner', rank: 'thetes' };
+}
+
+console.log('\nPOST /provisioning/instances/:id/restart:');
+
+await ta('an ACTIVE reachable project queues the restart and answers a receipt with no duration promise', async () => {
+  reset();
+  const r = await post('/provisioning/instances/18/restart');
+  assert.equal(r.status, 200, r.raw);
+  assert.equal(r.body.ok, true);
+  assert.equal(r.body.queued, true);
+  assert.equal(r.body.action, 'restart');
+  assert.equal(enqueueCalls.length, 1, 'exactly one enqueue');
+  assert.equal(enqueueCalls[0].requestedBy, 'api:self');
+  assert.doesNotMatch(r.body.message, PROMISE, `no wall-clock promise: ${r.body.message}`);
+  assert.equal(events.length, 1, 'the request is audited');
+  assert.equal(events[0].event, 'restart');
+  assert.equal(events[0].actor, 'api:self');
+});
+
+await ta('an already-open restart answers truthfully and records NO second event', async () => {
+  reset();
+  enqueueResult = { intent: { id: 9, action: 'restart', state: 'running' }, created: false };
+  const r = await post('/provisioning/instances/18/restart');
+  assert.equal(r.status, 200, r.raw);
+  assert.equal(r.body.queued, false);
+  assert.match(r.body.message, /already/i);
+  assert.equal(events.length, 0, 'no event for a request that queued nothing');
+});
+
+await ta('a foreign open intent → 409 intent_conflict naming the blocker', async () => {
+  reset();
+  enqueueResult = { intent: { id: 9, action: 'teardown', state: 'pending' }, created: false, conflict: 'open_intent' };
+  const r = await post('/provisioning/instances/18/restart');
+  assert.equal(r.status, 409, r.raw);
+  assert.equal(r.body.error.code, 'intent_conflict');
+  assert.equal(r.body.error.details.action, 'teardown');
+  assert.match(r.body.error.message, /teardown/, 'says WHAT is in the way');
+  assert.doesNotMatch(r.body.error.message, PROMISE);
+});
+
+await ta('every non-active state refuses with the action that IS available there — and queues nothing', async () => {
+  for (const [status, wanted] of [
+    ['torn_down', /stand it up again/i],
+    ['tearing_down', /stand it up again/i],
+    ['error', /stand it up again/i],
+    ['requested', /hasn’t finished standing up/i],
+    ['provisioning', /hasn’t finished standing up/i],
+  ]) {
+    reset({ status });
+    const r = await post('/provisioning/instances/18/restart');
+    assert.equal(r.status, 409, `${status}: ${r.raw}`);
+    assert.equal(r.body.error.code, 'not_restartable', status);
+    assert.equal(r.body.error.details.status, status, 'the refusal carries the state it read');
+    assert.match(r.body.error.message, wanted, `${status}: ${r.body.error.message}`);
+    assert.doesNotMatch(r.body.error.message, PROMISE, status);
+    assert.equal(enqueueCalls.length, 0, `${status} queued nothing`);
+  }
+});
+
+await ta('a dedicated droplet is refused HONESTLY and told the command to run — never queued a restart that cannot land', async () => {
+  reset({ hosting_shape: 'dedicated' });
+  const r = await post('/provisioning/instances/18/restart');
+  assert.equal(r.status, 409, r.raw);
+  assert.equal(r.body.error.code, 'restart_unreachable');
+  assert.equal(r.body.error.details.hosting_shape, 'dedicated');
+  assert.match(r.body.error.message, /systemctl restart spike/, 'names the exact command, on the machine that must run it');
+  assert.equal(enqueueCalls.length, 0, 'nothing queued');
+});
+
+await ta('the state gate is checked BEFORE the shape gate — a torn-down droplet hears about its state, not its shape', async () => {
+  reset({ status: 'torn_down', hosting_shape: 'dedicated' });
+  const r = await post('/provisioning/instances/18/restart');
+  assert.equal(r.body.error.code, 'not_restartable', 'the more useful refusal wins');
+});
+
+await ta("another builder's project is 404 — no existence leak", async () => {
+  reset();
+  builder = { id: 99, github_login: 'stranger', rank: 'thetes' };
+  const r = await post('/provisioning/instances/18/restart');
+  assert.equal(r.status, 404, r.raw);
+  assert.equal(r.body.error.code, 'instance_not_found');
+  assert.equal(enqueueCalls.length, 0);
+});
+
+await ta('an archon may restart any project, and the override is stamped with WHICH archon', async () => {
+  reset();
+  builder = { id: 42, github_login: 'arch', rank: 'archon' };
+  const r = await post('/provisioning/instances/18/restart');
+  assert.equal(r.status, 200, r.raw);
+  assert.equal(enqueueCalls[0].requestedBy, 'api:archon:42', 'the intent carries the override');
+  assert.equal(events[0].actor, 'api:archon:42', 'and so does the audit event');
+  assert.equal(events[0].ownerBuilderId, 7, 'the event still belongs to the owner');
+});
+
+await ta('a missing instance is 404, not a 500', async () => {
+  reset();
+  instance = null;
+  const r = await post('/provisioning/instances/18/restart');
+  assert.equal(r.status, 404, r.raw);
+});
+
+server.close();
+console.log(`\nprovisioning_restart: ${passed} passed, ${failed} failed`);
+process.exit(failed ? 1 : 0);

package/tests/provisioning_settings_apply.mjs

@@ -57,8 +57,13 @@ t("INTENT_ACTIONS includes 'settings-apply' (lockstep with the provisioning_010
   const sql = readFileSync(new URL('../modules/provisioning/migrations/provisioning_010_settings_apply.sql', import.meta.url), 'utf8');
   const m = /CHECK \(action IN \(([^)]*)\)\)/.exec(sql);
   assert.ok(m, 'the migration widens the action CHECK');
-  const inSql = m[1].split(',').map((s) => s.trim().replace(/^'|'$/g, '')).sort();
-  assert.deepEqual(inSql, [...provisioning.INTENT_ACTIONS].sort(), 'the CHECK and INTENT_ACTIONS carry the same set');
+  const inSql = m[1].split(',').map((s) => s.trim().replace(/^'|'$/g, ''));
+  assert.ok(inSql.includes('settings-apply'), 'this migration is the one that admitted it');
+  // Every value 010 permits is still a live action — the drift this guards. The
+  // strict set-equality lockstep lives with the NEWEST migration (a later one
+  // widens the CHECK again, and 010 is then deliberately a prefix of the set);
+  // asserting equality here would make every future action fail this test.
+  for (const a of inSql) assert.ok(provisioning.INTENT_ACTIONS.includes(a), `${a} is still an INTENT_ACTION`);
   assert.match(sql, /INSERT INTO schema_migrations \(version\) VALUES \('provisioning_010_settings_apply'\)/, 'self-registers');
 });
 
@@ -117,7 +122,7 @@ await ta('enqueueTeardown supersedes a PENDING settings push (a push into a proj
   const r = await provisioning.enqueueTeardown(db, 18, 'api:self');
   assert.equal(r.created, true);
   assert.equal(r.superseded, true, 'the queued push was cancelled so the teardown could take the slot');
-  assert.match(calls[0], /action IN \('provision', 'settings-apply'\) AND state = 'pending'/, 'pending-only, and only the two supersedable actions');
+  assert.match(calls[0], /action IN \('provision', 'settings-apply', 'restart'\) AND state = 'pending'/, 'pending-only, and only the supersedable actions');
 });
 
 // ── the route, over a real mounted router (stubbed data access) ─────────────────

package/tests/provisioning_teardown_intent.mjs

@@ -30,6 +30,9 @@ import http from 'node:http';
 const require = createRequire(import.meta.url);
 const provisioning = require('../modules/provisioning/provisioning.js');
 
+// The actions a teardown supersedes, mirroring enqueueTeardown's SQL.
+const SUPERSEDABLE = new Set(['provision', 'settings-apply', 'restart']);
+
 let passed = 0, failed = 0;
 function t(name, fn) {
   try { fn(); passed++; console.log(`  PASS  ${name}`); }
@@ -65,12 +68,13 @@ function fakeIntentDb(seed = []) {
       }
       if (/UPDATE provisioning_intents/.test(sql) && /state = 'error'/.test(sql)) {
         // The supersede's guard must live in the SQL itself: pending-only (a running
-        // intent is the runner's mid-flight work) and provision-or-settings-push only
-        // (task 1003140 widened it: a queued push into a project about to be torn
-        // down is dead work; an open teardown is left alone for the idempotent path).
-        assert.match(sql, /action IN \('provision', 'settings-apply'\) AND state = 'pending'/);
+        // intent is the runner's mid-flight work) and only the actions that become
+        // dead work in front of a teardown (task 1003140 widened it to the settings
+        // push, task 1003238 to the restart; an open teardown is left alone for the
+        // idempotent path).
+        assert.match(sql, /action IN \('provision', 'settings-apply', 'restart'\) AND state = 'pending'/);
         const hit = rows.filter((r) =>
-          String(r.instance_id) === String(params[0]) && (r.action === 'provision' || r.action === 'settings-apply') && r.state === 'pending');
+          String(r.instance_id) === String(params[0]) && SUPERSEDABLE.has(r.action) && r.state === 'pending');
         for (const r of hit) { r.state = 'error'; r.last_error = 'superseded: teardown requested'; }
         return { rows: hit.map((r) => ({ id: r.id })) };
       }