@agentpulselabs/cli-plugin-pulse 1.1.2 → 1.1.5

package/src/lib/cutoverRequest.js

@@ -0,0 +1,485 @@
+import { randomBytes, randomUUID } from 'node:crypto';
+import { chmod, link, open, rename, rm, unlink } from 'node:fs/promises';
+import path from 'node:path';
+import { setTimeout as sleep } from 'node:timers/promises';
+import { CUTOVER_FILE, STEPS } from './cutover.js';
+import { HubError } from './errors.js';
+import { readOwnerOnlyJson, writeOwnerOnlyJson } from './storage.js';
+
+export const CUTOVER_REQUEST_FILE = 'cutover.request';
+/** A request the daemon renamed off the disk to read it; one a crash leaves behind is refused on start, never run. */
+export const CUTOVER_CLAIMED_FILE = 'cutover.request.claimed';
+/** The request protocol a daemon serves, in every heartbeat; a daemon without it gets the cutover run in-process. */
+export const CUTOVER_PROTOCOL = 1;
+export const REQUEST_STALE_MS = 10 * 60_000;
+/** The exit code of a follow that cannot tell how the request ended (timed out, or the daemon stopped). */
+export const FOLLOW_UNKNOWN_EXIT = 2;
+const FUTURE_SLACK_MS = 60_000;
+const HEALTH_FRESH_MS = 60_000;
+const MODES = new Set(['cutover', 'rollback']);
+const SMOKE_CHANNEL = /^[A-Za-z0-9._-]{1,100}\/[A-Za-z0-9._-]{1,100}$/;
+const HISTORY = 50;
+const TICK_MS = 2_000;
+const ACCEPT_MS = 2 * 60_000;
+// The worst case of one request: holders (2 min), the final import (its 60 min wait), the reconcile (10 min), the
+// facade (90 s) and smoke, then the rollback that follows a failure (facade 90 s, spool 2 min, hub 90 s): about 80 min.
+const FOLLOW_MS = 2 * 3_600_000;
+const POLL_MS = 1_000;
+
+const iso = (ms) => new Date(ms).toISOString();
+const atOrAfter = (when, since) => Date.parse(when) >= Date.parse(since);
+
+function pidAlive(pid) {
+  try {
+    process.kill(pid, 0);
+    return true;
+  } catch (error) {
+    return error.code === 'EPERM';
+  }
+}
+
+/** Rollback steps from which the old hub may have hub.db open: a rollback that reached one is never run again. */
+const RELOADING = new Set(['hub_loading', 'hub_loaded', 'hub_serving']);
+
+/** The steps of the rollback journaled since `since` (this request's), by name. */
+function rollbackProgress(journal, since) {
+  const rollback = journal.rollback;
+  return rollback?.startedAt && atOrAfter(rollback.startedAt, since) ? rollback.steps.map((entry) => entry.name) : [];
+}
+
+/** The health state a finished request left. */
+function endState(request) {
+  const { outcome } = request;
+  if (outcome.state === 'done') return request.mode === 'cutover' ? 'serving' : 'rolled_back';
+  return outcome.rolledBack ? 'rolled_back' : 'failed';
+}
+
+/**
+ * The daemon's side of `pulse cutover` (CUT-03). It serves cutover.request on its own timer, beside the heartbeat, so
+ * the runner a cutover switches on can start while the cutover waits for it. One request runs at a time, in this
+ * process, with the real Cutover from `createCutover({ log })`, and only while `canTake()` (no restart is pending).
+ * A request is claimed by renaming it off the disk, checked, then recorded in cutover.json (`request`, and its id in
+ * `requests`) before the claimed file is deleted, so a crash never runs it twice; a stale, duplicate, malformed or
+ * pointless one is refused into `refusals`. On start, a request a crash cut off is finished as 1.1.2 would after a
+ * failure (the 20:05:57 DECISION): a cutover that had begun rolls back, a rollback runs again, unless either had begun
+ * loading the old hub (`hub_loading`): restoring hub.db under it would lose its writes, so boot only checks that the old
+ * hub holds the port and records rolled_back, or HUB_ROLLBACK_INTERRUPTED for a human. Each boot action is recorded in
+ * the journal's `recovery` and in health.
+ */
+export class CutoverService {
+  constructor({ directory, createCutover, canTake = () => true, now = Date.now, tickMs = TICK_MS, log = () => {} }) {
+    this.directory = directory;
+    this.createCutover = createCutover;
+    this.canTake = canTake;
+    this.now = now;
+    this.tickMs = tickMs;
+    this.log = log;
+    this.journalFile = path.join(directory, CUTOVER_FILE);
+    this.requestFile = path.join(directory, CUTOVER_REQUEST_FILE);
+    this.claimedFile = path.join(directory, CUTOVER_CLAIMED_FILE);
+    this.timer = null;
+    this.running = null;
+    this.recovered = false;
+    this.stopped = false;
+    this.active = false;
+    this.state = null;
+  }
+
+  start() {
+    this.stopped = false;
+    this.#schedule(0);
+  }
+
+  /** Stops taking requests. A cutover running now is not waited for: one cut off is finished by the next start. */
+  async stop() {
+    this.stopped = true;
+    clearTimeout(this.timer);
+    this.timer = null;
+  }
+
+  /** Whether a request is running: the daemon holds any restart until it ends. */
+  busy() {
+    return this.active;
+  }
+
+  /** `{ state, step, requestId }` for daemon-health.json, or null before any request. */
+  health() {
+    return this.state;
+  }
+
+  #schedule(waitMs) {
+    clearTimeout(this.timer);
+    if (this.stopped) return;
+    this.timer = setTimeout(() => {
+      this.timer = null;
+      void this.tick().finally(() => this.#schedule(this.tickMs));
+    }, waitMs);
+    this.timer.unref?.();
+  }
+
+  /** One pass: finish what a crash cut off (first pass only), then take one request if there is one. Never rejects. */
+  tick() {
+    this.running ??= this.#pass().finally(() => {
+      this.running = null;
+    });
+    return this.running;
+  }
+
+  async #pass() {
+    try {
+      if (!this.recovered) {
+        this.recovered = true;
+        await this.#recover();
+      }
+      if (this.stopped || !this.canTake()) return;
+      try {
+        await rename(this.requestFile, this.claimedFile);
+      } catch (error) {
+        if (error.code === 'ENOENT') return;
+        throw error;
+      }
+      await this.#take();
+    } catch (error) {
+      this.log('cutover_service_failed', { code: error.code || 'HUB_CUTOVER_SERVICE_FAILED', message: String(error.message).slice(0, 200) });
+    }
+  }
+
+  async #journal() {
+    return (await readOwnerOnlyJson(this.journalFile)) || { state: 'none', steps: {} };
+  }
+
+  async #claimed() {
+    try {
+      return { request: await readOwnerOnlyJson(this.claimedFile) };
+    } catch (error) {
+      return { request: null, unreadable: error.code === 'HUB_INVALID_CONFIG' ? 'not JSON' : `${error.code || 'unreadable'}: ${String(error.message).slice(0, 120)}` };
+    }
+  }
+
+  /** Why this request is refused, or null. */
+  #refusal(request, journal) {
+    if (!request || typeof request !== 'object') return 'invalid: not a JSON object';
+    if (typeof request.id !== 'string' || !/^[A-Za-z0-9._:-]{1,100}$/.test(request.id)) return 'invalid: no usable id';
+    if (!MODES.has(request.mode)) return `invalid: mode ${JSON.stringify(request.mode ?? null)} is neither cutover nor rollback`;
+    if (request.smokeChannel != null && !(typeof request.smokeChannel === 'string' && SMOKE_CHANNEL.test(request.smokeChannel))) {
+      return 'invalid: smokeChannel is not <repo>/<task>';
+    }
+    const at = Date.parse(request.requestedAt);
+    if (!Number.isFinite(at)) return 'invalid: no requestedAt';
+    if (at - this.now() > FUTURE_SLACK_MS) return `invalid: requestedAt ${request.requestedAt} is in the future`;
+    const ageMs = this.now() - at;
+    if (ageMs > REQUEST_STALE_MS) return `stale: requested ${Math.round(ageMs / 60_000)} min ago, more than ${REQUEST_STALE_MS / 60_000} min`;
+    if (journal.request?.id === request.id || (journal.requests || []).includes(request.id)) return `duplicate: request ${request.id} was already taken`;
+    const ended = journal.request?.outcome?.at;
+    if (ended && at < Date.parse(ended)) return `arrived while another request ran: requested ${request.requestedAt}, before request ${journal.request.id} ended at ${ended}`;
+    if (journal.state === 'running' && journal.runBy === 'cli' && Number.isInteger(journal.pid) && pidAlive(journal.pid)) {
+      return `busy: pid ${journal.pid} is running a cutover in-process (started ${journal.startedAt})`;
+    }
+    if (request.mode === 'rollback') {
+      if (journal.state === 'none' || journal.state === 'rolled_back') return `nothing to roll back: cutover.json is ${journal.state}`;
+      if ((journal.rollback?.steps || []).some((entry) => RELOADING.has(entry.name))) {
+        return 'a rollback already loaded the old hub again: another would restore hub.db under it; check it by hand';
+      }
+    }
+    return null;
+  }
+
+  async #take() {
+    const { request, unreadable } = await this.#claimed();
+    const journal = await this.#journal();
+    const reason = unreadable ? `invalid: ${unreadable}` : this.#refusal(request, journal);
+    if (reason) {
+      await this.#refuse(request, reason, journal);
+      await rm(this.claimedFile, { force: true });
+      return;
+    }
+    // A restart requested while this was read: the request goes back for the next process, or is refused if it can't.
+    if (this.stopped || !this.canTake()) {
+      const back = await link(this.claimedFile, this.requestFile).then(() => true, () => false);
+      if (!back) await this.#refuse(request, 'the Pulse daemon is restarting; ask again once it is back', journal);
+      await rm(this.claimedFile, { force: true });
+      return;
+    }
+    this.active = true;
+    try {
+      const accepted = {
+        id: request.id, mode: request.mode, requestedAt: request.requestedAt, requestedBy: request.requestedBy ?? null,
+        ...(request.mode === 'cutover' && request.smokeChannel ? { smokeChannel: request.smokeChannel } : {}),
+        acceptedAt: iso(this.now()),
+      };
+      await writeOwnerOnlyJson(this.journalFile, { ...journal, request: accepted, requests: [...(journal.requests || []), accepted.id].slice(-HISTORY) });
+      await rm(this.claimedFile, { force: true });
+      this.log('cutover_request_accepted', { id: accepted.id, mode: accepted.mode });
+      await this.#perform(accepted, accepted.mode);
+    } finally {
+      this.active = false;
+    }
+  }
+
+  async #refuse(request, reason, journal) {
+    const refusal = {
+      id: typeof request?.id === 'string' ? request.id : null, mode: request?.mode ?? null, requestedAt: request?.requestedAt ?? null,
+      reason, at: iso(this.now()),
+    };
+    await writeOwnerOnlyJson(this.journalFile, { ...journal, refusals: [...(journal.refusals || []), refusal].slice(-HISTORY) });
+    this.state = { state: 'refused', requestId: refusal.id, reason };
+    this.log('cutover_request_refused', { id: refusal.id, reason });
+  }
+
+  /**
+   * Runs `action` (cutover or rollback) for an accepted request and records its outcome on the journal. `asOutcome`
+   * is what a rollback that finishes a cut-off cutover reports, rolled back, instead of a plain done; `recovered` names
+   * the boot action that started it, for health.
+   */
+  async #perform(request, action, { asOutcome = null, recovered = null } = {}) {
+    const tag = recovered ? { recovered } : {};
+    this.state = { state: 'running', mode: action, step: null, requestId: request.id, since: iso(this.now()), ...tag };
+    const log = (event, details = {}) => {
+      if (event === 'cutover_step') this.state = { ...this.state, step: details.step };
+      if (event === 'rollback_step') this.state = { ...this.state, phase: 'rollback', step: details.name };
+      this.log(event, details);
+    };
+    // Health says how it ended before the journal does, so whoever sees the outcome sees the same in health.
+    try {
+      const cutover = await this.createCutover({ log });
+      if (action === 'cutover') {
+        await cutover.run({ smokeChannel: request.smokeChannel ?? null, request });
+        this.state = { state: 'serving', step: this.state.step, requestId: request.id, ...tag };
+        await this.#finish(request.id, { state: 'done' });
+        return;
+      }
+      const report = await cutover.rollback({ reason: asOutcome?.code ?? 'asked' });
+      this.state = { state: 'rolled_back', step: this.state.step, requestId: request.id, ...tag };
+      await this.#finish(request.id, asOutcome ? { ...asOutcome, rolledBack: true } : { state: 'done', message: report.message });
+    } catch (error) {
+      const code = error.code || 'HUB_CUTOVER_FAILED';
+      this.state = { state: error.rollback ? 'rolled_back' : 'failed', step: this.state.step, requestId: request.id, code, ...tag };
+      this.log('cutover_request_failed', { id: request.id, code });
+      await this.#finish(request.id, {
+        state: 'failed', code, message: String(error.message).slice(0, 500), rolledBack: Boolean(error.rollback),
+        ...(error.details?.checks ? { checks: error.details.checks } : {}),
+      });
+    }
+  }
+
+  /** Records the request's outcome on the journal, with `extra` journal fields (a boot recovery's). */
+  async #finish(id, outcome, extra = {}) {
+    const journal = await this.#journal();
+    const request = journal.request?.id === id ? journal.request : { id };
+    await writeOwnerOnlyJson(this.journalFile, { ...journal, ...extra, request: { ...request, outcome: { ...outcome, at: iso(this.now()) } } });
+  }
+
+  /** A boot recovery that ends the request without running anything: the outcome and what boot did, in both files. */
+  async #recovered(request, action, outcome, state, extra = {}) {
+    const recovery = { at: iso(this.now()), requestId: request.id, action };
+    this.state = { state, requestId: request.id, recovered: action, ...(outcome.code ? { code: outcome.code } : {}) };
+    this.log('cutover_recovered', { id: request.id, action });
+    await this.#finish(request.id, outcome, { ...extra, recovery });
+  }
+
+  /** On start: a claim a crash left is refused; a request that was running is finished (see the class). */
+  async #recover() {
+    const claimed = await this.#claimed();
+    if (claimed.request || claimed.unreadable) {
+      const journal = await this.#journal();
+      if (!(claimed.request && (journal.requests || []).includes(claimed.request.id))) {
+        await this.#refuse(claimed.request, 'interrupted: the daemon stopped while it was reading this request', journal);
+      }
+      await rm(this.claimedFile, { force: true });
+    }
+    const journal = await this.#journal();
+    const request = journal.request;
+    if (!request?.id) return;
+    if (request.outcome) {
+      this.state = { state: endState(request), requestId: request.id, ...(request.outcome.code ? { code: request.outcome.code } : {}) };
+      return;
+    }
+    const since = request.acceptedAt;
+    const began = Boolean(journal.startedAt) && atOrAfter(journal.startedAt, since);
+    const rolledBack = journal.state === 'rolled_back' && atOrAfter(journal.rolledBackAt, since);
+    const reloading = rollbackProgress(journal, since).some((name) => RELOADING.has(name));
+    const failure = journal.failure ?? { code: 'HUB_CUTOVER_INTERRUPTED', message: 'The Pulse daemon stopped while it ran this cutover.' };
+    // What a cutover that ended rolled back reports; a rollback request that ended so is simply done.
+    const rolledBackOutcome = (message) => (request.mode === 'rollback'
+      ? { state: 'done', message }
+      : { state: 'failed', code: failure.code, message: failure.message, rolledBack: true });
+    this.active = true;
+    try {
+      if (request.mode === 'cutover' && journal.state === 'done' && began) return await this.#recovered(request, 'finished_before_stop', { state: 'done' }, 'serving');
+      if (rolledBack) return await this.#recovered(request, 'rolled_back_before_stop', rolledBackOutcome(journal.rollback?.message ?? null), 'rolled_back');
+      if (request.mode === 'cutover' && !(journal.state === 'running' && began)) {
+        return await this.#recovered(request, 'never_began', {
+          state: 'failed', code: 'HUB_CUTOVER_INTERRUPTED', message: 'The Pulse daemon stopped before this cutover began; nothing was changed.', rolledBack: false,
+        }, 'failed');
+      }
+      if (reloading) return await this.#verifyReloaded(request, journal, rolledBackOutcome);
+      if (request.mode === 'cutover') {
+        const cutFailure = journal.failure ?? {
+          step: STEPS.find((name) => !journal.steps?.[name]) ?? null, code: 'HUB_CUTOVER_INTERRUPTED',
+          message: 'The Pulse daemon stopped while it ran this cutover; it rolled back on start.',
+        };
+        const recovery = { at: iso(this.now()), requestId: request.id, action: 'rolled_back_on_start' };
+        await writeOwnerOnlyJson(this.journalFile, { ...journal, failure: cutFailure, recovery });
+        this.log('cutover_interrupted', { id: request.id, mode: 'cutover', step: cutFailure.step });
+        return await this.#perform(request, 'rollback', { asOutcome: { state: 'failed', code: 'HUB_CUTOVER_INTERRUPTED', message: cutFailure.message }, recovered: 'rolled_back_on_start' });
+      }
+      await writeOwnerOnlyJson(this.journalFile, { ...journal, recovery: { at: iso(this.now()), requestId: request.id, action: 'rollback_run_again' } });
+      this.log('cutover_interrupted', { id: request.id, mode: 'rollback' });
+      return await this.#perform(request, 'rollback', { recovered: 'rollback_run_again' });
+    } finally {
+      this.active = false;
+    }
+  }
+
+  /**
+   * The 20:05:57 DECISION: a rollback that had begun loading the old hub is never run again (it would restore hub.db
+   * under the running hub). Boot only checks that the old hub holds the port, and records rolled_back if it does.
+   */
+  async #verifyReloaded(request, journal, rolledBackOutcome) {
+    this.state = { state: 'running', mode: 'verify', step: 'hub_loading', requestId: request.id, since: iso(this.now()) };
+    let onPort;
+    try {
+      onPort = await (await this.createCutover({ log: this.log })).verifyRolledBack();
+    } catch (error) {
+      onPort = { ok: false, detail: `the check failed: ${error.code || 'error'}: ${String(error.message).slice(0, 200)}` };
+    }
+    if (onPort.ok) {
+      const at = iso(this.now());
+      const message = `Rolled back to the local hub: the Pulse daemon restarted after the old hub was loaded again, and found it on the port (${onPort.detail}).`;
+      return this.#recovered(request, 'verified_old_hub', rolledBackOutcome(message), 'rolled_back', {
+        state: 'rolled_back', rolledBackAt: at, rollback: { ...journal.rollback, recoveredOnBoot: true, verified: onPort.detail, message },
+      });
+    }
+    return this.#recovered(request, 'old_hub_not_serving', {
+      state: 'failed', code: 'HUB_ROLLBACK_INTERRUPTED', rolledBack: false,
+      message: `The Pulse daemon stopped after its rollback began loading the old hub, and the old hub does not hold the port now (${onPort.detail}). It is not rolled back twice, which would restore hub.db under a hub that may have it open: check the hub by hand.`,
+    }, 'failed');
+  }
+}
+
+/** Whether the daemon behind this health file is alive and serves cutover.request. */
+export function takesCutoverRequests(health, now = Date.now()) {
+  const ageMs = now - Date.parse(health?.heartbeatAt);
+  return ageMs <= HEALTH_FRESH_MS && Number(health?.cutoverProtocol) >= CUTOVER_PROTOCOL;
+}
+
+/** Creates `file` owner-only with `value`, whole or not at all, and only if it does not exist yet. */
+async function createExclusive(file, value) {
+  const temporary = path.join(path.dirname(file), `.${path.basename(file)}.${process.pid}.${randomBytes(6).toString('hex')}.tmp`);
+  const handle = await open(temporary, 'wx', 0o600);
+  try {
+    await handle.writeFile(`${JSON.stringify(value, null, 2)}\n`, 'utf8');
+    await handle.sync();
+  } finally {
+    await handle.close();
+  }
+  try {
+    await chmod(temporary, 0o600);
+    await link(temporary, file);
+    return true;
+  } catch (error) {
+    if (error.code === 'EEXIST') return false;
+    throw error;
+  } finally {
+    await rm(temporary, { force: true });
+  }
+}
+
+/** Writes an owner-only cutover.request for the daemon, exclusively: one request at a time. */
+export async function writeCutoverRequest({ configDir, mode, smokeChannel = null, health = null, now = Date.now, id = randomUUID(), requestedBy = null }) {
+  const pending = (detail) => new HubError('HUB_CUTOVER_PENDING', `Another cutover request is ${detail}; wait for it to finish (see ${path.join(configDir, CUTOVER_FILE)}).`);
+  if (health?.cutover?.state === 'running') throw pending(`running in the Pulse daemon (${health.cutover.requestId}${health.cutover.step ? `, at ${health.cutover.step}` : ''})`);
+  const request = {
+    id, mode, requestedAt: iso(now()),
+    requestedBy: requestedBy ?? { pid: process.pid, session: process.env.CURSOR_CONVERSATION_ID || null },
+    ...(smokeChannel ? { smokeChannel } : {}),
+  };
+  if (!(await createExclusive(path.join(configDir, CUTOVER_REQUEST_FILE), request))) throw pending('waiting for the Pulse daemon');
+  return request;
+}
+
+/**
+ * Follows a request through cutover.json, printing each step as `log(event, details)` exactly as an in-process run
+ * logs it. It resolves with the journal (a cutover) or the rollback report, or throws: the journal's code for a run
+ * that failed (or rolled back), HUB_CUTOVER_REFUSED, or HUB_CUTOVER_NOT_ACCEPTED when nobody took it within
+ * `acceptMs` (it is withdrawn). It ends without an outcome, with `exit` FOLLOW_UNKNOWN_EXIT, when the daemon's heartbeat
+ * goes stale (it finishes the request when it starts again) or after `timeoutMs`, the worst case of a run.
+ */
+export async function followCutoverRequest({
+  configDir, request, log = () => {}, now = Date.now, wait = sleep, pollMs = POLL_MS, acceptMs = ACCEPT_MS, timeoutMs = FOLLOW_MS,
+  healthFreshMs = HEALTH_FRESH_MS,
+}) {
+  const journalFile = path.join(configDir, CUTOVER_FILE);
+  const requestFile = path.join(configDir, CUTOVER_REQUEST_FILE);
+  const healthFile = path.join(configDir, 'daemon-health.json');
+  const started = now();
+  const printed = { steps: new Set(), failure: false, rollback: 0 };
+  let withDaemon = false;
+  const print = (journal) => {
+    const since = journal.request.acceptedAt;
+    if (request.mode === 'cutover' && journal.startedAt && atOrAfter(journal.startedAt, since)) {
+      for (const name of STEPS) {
+        if (!journal.steps?.[name] || printed.steps.has(name)) continue;
+        printed.steps.add(name);
+        const { at: _at, ...detail } = journal.steps[name];
+        log('cutover_step', { step: name, ...detail });
+      }
+      if (journal.failure && !printed.failure) {
+        printed.failure = true;
+        log('cutover_failed', journal.failure);
+      }
+    }
+    if (rollbackProgress(journal, since).length) {
+      for (const entry of journal.rollback.steps.slice(printed.rollback)) log('rollback_step', entry);
+      printed.rollback = journal.rollback.steps.length;
+    }
+  };
+  const unknown = (code, message) => new HubError(code, message, { exit: FOLLOW_UNKNOWN_EXIT });
+  for (;;) {
+    const journal = await readOwnerOnlyJson(journalFile).catch(() => null);
+    const refused = journal?.refusals?.find((entry) => entry.id === request.id);
+    if (refused) throw new HubError('HUB_CUTOVER_REFUSED', `The Pulse daemon refused request ${request.id}: ${refused.reason}`, { details: { refused } });
+    const health = await readOwnerOnlyJson(healthFile).catch(() => null);
+    // A long synchronous stretch (a rescan) can delay a heartbeat; a daemon whose pid is alive gets three windows.
+    const ageMs = now() - Date.parse(health?.heartbeatAt);
+    const stale = !(ageMs <= healthFreshMs) && (!(ageMs <= 3 * healthFreshMs) || !(Number.isInteger(health?.pid) && pidAlive(health.pid)));
+    if (journal?.request?.id === request.id) {
+      print(journal);
+      const outcome = journal.request.outcome;
+      if (outcome?.state === 'done') return request.mode === 'cutover' ? journal : journal.rollback;
+      if (outcome) {
+        throw Object.assign(
+          new HubError(outcome.code || 'HUB_CUTOVER_FAILED', outcome.message || 'the cutover failed', { details: { ...(outcome.checks ? { checks: outcome.checks } : {}), rolledBack: Boolean(outcome.rolledBack) } }),
+          outcome.rolledBack ? { rollback: journal.rollback ?? null } : {},
+        );
+      }
+      if (stale) {
+        throw unknown('HUB_CUTOVER_DAEMON_STOPPED', `The Pulse daemon stopped (last heartbeat ${health?.heartbeatAt ?? 'never'}) while it ran request ${request.id}. When it starts again it finishes the request (a cutover that had begun rolls back); follow it in ${journalFile}.`);
+      }
+    } else if (!withDaemon && (stale || now() - started >= acceptMs)) {
+      // Withdrawn only while it is still ours on the disk: gone means the daemon claimed it, so it is followed.
+      const pending = await readOwnerOnlyJson(requestFile).catch(() => null);
+      let withdrawn = false;
+      if (pending?.id === request.id) {
+        try {
+          await unlink(requestFile);
+          withdrawn = true;
+        } catch (error) {
+          if (error.code !== 'ENOENT') throw error;
+        }
+      }
+      if (withdrawn) {
+        throw new HubError('HUB_CUTOVER_NOT_ACCEPTED', stale
+          ? `The Pulse daemon is not running (last heartbeat ${health?.heartbeatAt ?? 'never'}), so request ${request.id} was withdrawn. Start it with \`agentpulse pulse start\`.`
+          : `The Pulse daemon did not take request ${request.id} within ${Math.round(acceptMs / 1000)} s, so it was withdrawn. Check it with \`agentpulse pulse status\`.`);
+      }
+      withDaemon = true;
+    } else if (withDaemon && stale) {
+      throw unknown('HUB_CUTOVER_DAEMON_STOPPED', `The Pulse daemon stopped (last heartbeat ${health?.heartbeatAt ?? 'never'}) after it claimed request ${request.id}; when it starts again it refuses or finishes it. Follow it in ${journalFile}.`);
+    }
+    if (now() - started >= timeoutMs) {
+      throw unknown('HUB_CUTOVER_FOLLOW_TIMEOUT', `Request ${request.id} is still running in the Pulse daemon after ${Math.round(timeoutMs / 60_000)} min; follow it in ${journalFile} or with \`agentpulse pulse status\`.`);
+    }
+    await wait(pollMs);
+  }
+}

package/src/lib/cutoverWiring.js

@@ -109,14 +109,17 @@ export function portProbes(tokenFor) {
   };
 }
 
-/** The production seams of `pulse cutover`: launchd, lsof, the Phase 2 import and reconcile, and the local port. */
-export async function createCutover({ configDir, log = () => {}, port = 8890 }) {
+/**
+ * The production seams of `pulse cutover`: launchd, lsof, the Phase 2 import and reconcile, and the local port.
+ * `serve` is the daemon's own runner switch when the daemon runs it (CUT-03); in-process it is left to the heartbeat.
+ */
+export async function createCutover({ configDir, log = () => {}, port = 8890, serve = undefined }) {
   const config = await readOwnerOnlyJson(path.join(configDir, 'config.json'));
   if (!config) throw new HubError('HUB_NOT_INSTALLED', 'Pulse is not installed here; run `agentpulse pulse install`');
   const hubHome = config.cloudFirst?.hubHome || config.mirror?.hub?.dir || (await discoverLocalHub({ config })).dir;
   const tokenFor = () => hubToken(hubHome);
   return new Cutover({
-    configDir, hubHome, port, log,
+    configDir, hubHome, port, log, serve,
     services: new ServiceManager({}),
     probe: { holders: (files) => lsofHolders(files), portOwners: (p) => lsofPortOwners(p) },
     cloudFirst: new CloudFirstCommands({ configDir }),

package/src/lib/daemonProcess.js

@@ -1,4 +1,6 @@
 import path from 'node:path';
+import { CUTOVER_PROTOCOL, CutoverService } from './cutoverRequest.js';
+import { createCutover } from './cutoverWiring.js';
 import { createProductionDependencies } from './daemonComposition.js';
 import { DaemonLock } from './daemonLock.js';
 import { createHeartbeat, MIRROR_UPDATE_STEP, TERMINAL_STATES } from './daemonLoop.js';
@@ -60,7 +62,8 @@ const inside = (child, parent) => {
  * - start_failed: it could not start while it was holding restarts back.
  * A code or request restart drains (no new claim; running jobs finish and settle) for up to drainMs; a stall aborts at
  * once. Every trigger goes through one check that runs each tickMs. While the daemon has restarted too often it holds
- * the restart (health says restart_loop) until the window clears. SIGTERM always wins and exits 0.
+ * the restart (health says restart_loop) until the window clears, and while it runs a cutover request (CUT-03) it holds
+ * it until the cutover ends, whose own deadlines bound it. SIGTERM always wins and exits 0.
  */
 export async function runDaemon({
   directory,
@@ -74,6 +77,8 @@ export async function runDaemon({
   createMirror = (options) => new MirrorController(options),
   createShadow = (options) => new ShadowService(options),
   createRunner = (options) => new RunnerService(options),
+  createCutoverService = (options) => new CutoverService(options),
+  cutoverFactory = createCutover,
   dependencyOptions = {},
   runtimeOptions = {},
   lockOptions = {},
@@ -85,7 +90,7 @@ export async function runDaemon({
   const lock = new DaemonLock({ directory, ...lockOptions });
   await lock.acquire();
   // The first cycle (token exchange, reconnect, claim) can outlast `pulse start`'s wait; say we are up before it.
-  await lock.heartbeat({ state: 'starting' });
+  await lock.heartbeat({ state: 'starting', cutoverProtocol: CUTOVER_PROTOCOL });
   let startingHealthAt = Date.now();
   const guard = new RestartGuard({ directory, now });
   await guard.recordStart();
@@ -99,6 +104,11 @@ export async function runDaemon({
   let runtime = null;
   let shadow = null;
   let runnerService = null;
+  let cutoverService = null;
+  let cutoverDeferralLogged = false;
+  // Every read of config.json is numbered, so the runner never applies one older than it already has (CUT-03).
+  let configGeneration = 0;
+  let heartbeatGeneration = 0;
   let stopServices = async () => {};
   let timer = null;
   let watcher = null;
@@ -207,6 +217,14 @@ export async function runDaemon({
       const requestedAt = await consumeRestartRequest(directory);
       if (requestedAt !== null && requestedAt < bootedAt) log('restart_request_satisfied', { requestedAt: new Date(requestedAt).toISOString() });
       else if (requestedAt !== null) pending ??= { reason: 'request' };
+      if (pending && cutoverService?.busy()) {
+        if (!cutoverDeferralLogged) {
+          cutoverDeferralLogged = true;
+          log('restart_deferred', { reason: pending.reason, cutover: cutoverService.health()?.requestId ?? null });
+        }
+        return;
+      }
+      cutoverDeferralLogged = false;
       if (pending) restart(pending.reason);
     } catch (error) {
       log('restart_check_failed', { code: error.code || 'HUB_RESTART_CHECK_FAILED' });
@@ -217,11 +235,15 @@ export async function runDaemon({
 
   const extraHealth = () => {
     const loop = degraded();
+    const cutover = cutoverService?.health();
     return {
       ...(loop ? { degraded: loop } : {}),
       ...(restarting ? { restarting: { ...restarting, draining: runtime?.activeJobIds() ?? [] } } : {}),
+      ...(pending && cutoverService?.busy() ? { restartDeferred: { reason: pending.reason, untilCutoverEnds: cutover?.requestId ?? null } } : {}),
       ...(shadow ? { shadow: shadow.health() } : {}),
       ...(runnerService ? { runner: runnerService.health() } : {}),
+      cutoverProtocol: CUTOVER_PROTOCOL,
+      ...(cutover ? { cutover } : {}),
     };
   };
 
@@ -325,19 +347,34 @@ export async function runDaemon({
     shadow = createShadow({ directory, transportFactory: production.transportFactory, tokenStore: production.tokenStore, log: mirrorLog });
     runnerService = createRunner({ directory, transportFactory: production.transportFactory, log });
     heartbeat = createHeartbeat({
-      readConfig: () => readOwnerOnlyJson(configFile),
+      readConfig: () => {
+        heartbeatGeneration = ++configGeneration;
+        return readOwnerOnlyJson(configFile);
+      },
       runtime,
       mirror,
       shadow,
-      runner: runnerService,
+      runner: { update: (config) => runnerService.update(config, { generation: heartbeatGeneration }) },
       lock,
       ledger,
       ownerStore,
       progress: cycleProgress,
       extraHealth,
     });
+    // A cutover request runs here, beside the heartbeat; its facade step starts this daemon's runner at once.
+    const serve = async () => {
+      const generation = ++configGeneration;
+      runnerService.update(await readOwnerOnlyJson(configFile), { generation });
+    };
+    cutoverService = createCutoverService({
+      directory,
+      log,
+      canTake: () => !exiting && !restarting && !pending,
+      createCutover: ({ log: stepLog }) => cutoverFactory({ configDir: directory, log: stepLog, serve }),
+    });
+    cutoverService.start();
     stopServices = async ({ abortCode }) => {
-      const results = await Promise.allSettled([mirror.stop(), shadow.stop(), runnerService.stop(), runtime.shutdown({ abortCode })]);
+      const results = await Promise.allSettled([cutoverService.stop(), mirror.stop(), shadow.stop(), runnerService.stop(), runtime.shutdown({ abortCode })]);
       const failed = results.find((result) => result.status === 'rejected');
       if (failed) throw failed.reason;
     };
@@ -367,7 +404,7 @@ export async function runDaemon({
       // A cycle that ends after shutdown released the lock has no health left to report.
       if (exiting) return;
       await Promise.resolve()
-        .then(() => lock.heartbeat({ state: 'error', code: error.code || 'HUB_DAEMON_LOOP_FAILED' }))
+        .then(() => lock.heartbeat({ state: 'error', code: error.code || 'HUB_DAEMON_LOOP_FAILED', ...extraHealth() }))
         .catch(() => {});
       wait = t.errorRetryMs;
     } finally {