@agentpulselabs/cli-plugin-pulse 1.0.0 → 1.1.3

package/src/lib/cutoverRequest.js

@@ -0,0 +1,485 @@
+import { randomBytes, randomUUID } from 'node:crypto';
+import { chmod, link, open, rename, rm, unlink } from 'node:fs/promises';
+import path from 'node:path';
+import { setTimeout as sleep } from 'node:timers/promises';
+import { CUTOVER_FILE, STEPS } from './cutover.js';
+import { HubError } from './errors.js';
+import { readOwnerOnlyJson, writeOwnerOnlyJson } from './storage.js';
+
+export const CUTOVER_REQUEST_FILE = 'cutover.request';
+/** A request the daemon renamed off the disk to read it; one a crash leaves behind is refused on start, never run. */
+export const CUTOVER_CLAIMED_FILE = 'cutover.request.claimed';
+/** The request protocol a daemon serves, in every heartbeat; a daemon without it gets the cutover run in-process. */
+export const CUTOVER_PROTOCOL = 1;
+export const REQUEST_STALE_MS = 10 * 60_000;
+/** The exit code of a follow that cannot tell how the request ended (timed out, or the daemon stopped). */
+export const FOLLOW_UNKNOWN_EXIT = 2;
+const FUTURE_SLACK_MS = 60_000;
+const HEALTH_FRESH_MS = 60_000;
+const MODES = new Set(['cutover', 'rollback']);
+const SMOKE_CHANNEL = /^[A-Za-z0-9._-]{1,100}\/[A-Za-z0-9._-]{1,100}$/;
+const HISTORY = 50;
+const TICK_MS = 2_000;
+const ACCEPT_MS = 2 * 60_000;
+// The worst case of one request: holders (2 min), the final import (its 60 min wait), the reconcile (10 min), the
+// facade (90 s) and smoke, then the rollback that follows a failure (facade 90 s, spool 2 min, hub 90 s): about 80 min.
+const FOLLOW_MS = 2 * 3_600_000;
+const POLL_MS = 1_000;
+
+const iso = (ms) => new Date(ms).toISOString();
+const atOrAfter = (when, since) => Date.parse(when) >= Date.parse(since);
+
+function pidAlive(pid) {
+  try {
+    process.kill(pid, 0);
+    return true;
+  } catch (error) {
+    return error.code === 'EPERM';
+  }
+}
+
+/** Rollback steps from which the old hub may have hub.db open: a rollback that reached one is never run again. */
+const RELOADING = new Set(['hub_loading', 'hub_loaded', 'hub_serving']);
+
+/** The steps of the rollback journaled since `since` (this request's), by name. */
+function rollbackProgress(journal, since) {
+  const rollback = journal.rollback;
+  return rollback?.startedAt && atOrAfter(rollback.startedAt, since) ? rollback.steps.map((entry) => entry.name) : [];
+}
+
+/** The health state a finished request left. */
+function endState(request) {
+  const { outcome } = request;
+  if (outcome.state === 'done') return request.mode === 'cutover' ? 'serving' : 'rolled_back';
+  return outcome.rolledBack ? 'rolled_back' : 'failed';
+}
+
+/**
+ * The daemon's side of `pulse cutover` (CUT-03). It serves cutover.request on its own timer, beside the heartbeat, so
+ * the runner a cutover switches on can start while the cutover waits for it. One request runs at a time, in this
+ * process, with the real Cutover from `createCutover({ log })`, and only while `canTake()` (no restart is pending).
+ * A request is claimed by renaming it off the disk, checked, then recorded in cutover.json (`request`, and its id in
+ * `requests`) before the claimed file is deleted, so a crash never runs it twice; a stale, duplicate, malformed or
+ * pointless one is refused into `refusals`. On start, a request a crash cut off is finished as 1.1.2 would after a
+ * failure (the 20:05:57 DECISION): a cutover that had begun rolls back, a rollback runs again, unless either had begun
+ * loading the old hub (`hub_loading`): restoring hub.db under it would lose its writes, so boot only checks that the old
+ * hub holds the port and records rolled_back, or HUB_ROLLBACK_INTERRUPTED for a human. Each boot action is recorded in
+ * the journal's `recovery` and in health.
+ */
+export class CutoverService {
+  constructor({ directory, createCutover, canTake = () => true, now = Date.now, tickMs = TICK_MS, log = () => {} }) {
+    this.directory = directory;
+    this.createCutover = createCutover;
+    this.canTake = canTake;
+    this.now = now;
+    this.tickMs = tickMs;
+    this.log = log;
+    this.journalFile = path.join(directory, CUTOVER_FILE);
+    this.requestFile = path.join(directory, CUTOVER_REQUEST_FILE);
+    this.claimedFile = path.join(directory, CUTOVER_CLAIMED_FILE);
+    this.timer = null;
+    this.running = null;
+    this.recovered = false;
+    this.stopped = false;
+    this.active = false;
+    this.state = null;
+  }
+
+  start() {
+    this.stopped = false;
+    this.#schedule(0);
+  }
+
+  /** Stops taking requests. A cutover running now is not waited for: one cut off is finished by the next start. */
+  async stop() {
+    this.stopped = true;
+    clearTimeout(this.timer);
+    this.timer = null;
+  }
+
+  /** Whether a request is running: the daemon holds any restart until it ends. */
+  busy() {
+    return this.active;
+  }
+
+  /** `{ state, step, requestId }` for daemon-health.json, or null before any request. */
+  health() {
+    return this.state;
+  }
+
+  #schedule(waitMs) {
+    clearTimeout(this.timer);
+    if (this.stopped) return;
+    this.timer = setTimeout(() => {
+      this.timer = null;
+      void this.tick().finally(() => this.#schedule(this.tickMs));
+    }, waitMs);
+    this.timer.unref?.();
+  }
+
+  /** One pass: finish what a crash cut off (first pass only), then take one request if there is one. Never rejects. */
+  tick() {
+    this.running ??= this.#pass().finally(() => {
+      this.running = null;
+    });
+    return this.running;
+  }
+
+  async #pass() {
+    try {
+      if (!this.recovered) {
+        this.recovered = true;
+        await this.#recover();
+      }
+      if (this.stopped || !this.canTake()) return;
+      try {
+        await rename(this.requestFile, this.claimedFile);
+      } catch (error) {
+        if (error.code === 'ENOENT') return;
+        throw error;
+      }
+      await this.#take();
+    } catch (error) {
+      this.log('cutover_service_failed', { code: error.code || 'HUB_CUTOVER_SERVICE_FAILED', message: String(error.message).slice(0, 200) });
+    }
+  }
+
+  async #journal() {
+    return (await readOwnerOnlyJson(this.journalFile)) || { state: 'none', steps: {} };
+  }
+
+  async #claimed() {
+    try {
+      return { request: await readOwnerOnlyJson(this.claimedFile) };
+    } catch (error) {
+      return { request: null, unreadable: error.code === 'HUB_INVALID_CONFIG' ? 'not JSON' : `${error.code || 'unreadable'}: ${String(error.message).slice(0, 120)}` };
+    }
+  }
+
+  /** Why this request is refused, or null. */
+  #refusal(request, journal) {
+    if (!request || typeof request !== 'object') return 'invalid: not a JSON object';
+    if (typeof request.id !== 'string' || !/^[A-Za-z0-9._:-]{1,100}$/.test(request.id)) return 'invalid: no usable id';
+    if (!MODES.has(request.mode)) return `invalid: mode ${JSON.stringify(request.mode ?? null)} is neither cutover nor rollback`;
+    if (request.smokeChannel != null && !(typeof request.smokeChannel === 'string' && SMOKE_CHANNEL.test(request.smokeChannel))) {
+      return 'invalid: smokeChannel is not <repo>/<task>';
+    }
+    const at = Date.parse(request.requestedAt);
+    if (!Number.isFinite(at)) return 'invalid: no requestedAt';
+    if (at - this.now() > FUTURE_SLACK_MS) return `invalid: requestedAt ${request.requestedAt} is in the future`;
+    const ageMs = this.now() - at;
+    if (ageMs > REQUEST_STALE_MS) return `stale: requested ${Math.round(ageMs / 60_000)} min ago, more than ${REQUEST_STALE_MS / 60_000} min`;
+    if (journal.request?.id === request.id || (journal.requests || []).includes(request.id)) return `duplicate: request ${request.id} was already taken`;
+    const ended = journal.request?.outcome?.at;
+    if (ended && at < Date.parse(ended)) return `arrived while another request ran: requested ${request.requestedAt}, before request ${journal.request.id} ended at ${ended}`;
+    if (journal.state === 'running' && journal.runBy === 'cli' && Number.isInteger(journal.pid) && pidAlive(journal.pid)) {
+      return `busy: pid ${journal.pid} is running a cutover in-process (started ${journal.startedAt})`;
+    }
+    if (request.mode === 'rollback') {
+      if (journal.state === 'none' || journal.state === 'rolled_back') return `nothing to roll back: cutover.json is ${journal.state}`;
+      if ((journal.rollback?.steps || []).some((entry) => RELOADING.has(entry.name))) {
+        return 'a rollback already loaded the old hub again: another would restore hub.db under it; check it by hand';
+      }
+    }
+    return null;
+  }
+
+  async #take() {
+    const { request, unreadable } = await this.#claimed();
+    const journal = await this.#journal();
+    const reason = unreadable ? `invalid: ${unreadable}` : this.#refusal(request, journal);
+    if (reason) {
+      await this.#refuse(request, reason, journal);
+      await rm(this.claimedFile, { force: true });
+      return;
+    }
+    // A restart requested while this was read: the request goes back for the next process, or is refused if it can't.
+    if (this.stopped || !this.canTake()) {
+      const back = await link(this.claimedFile, this.requestFile).then(() => true, () => false);
+      if (!back) await this.#refuse(request, 'the Pulse daemon is restarting; ask again once it is back', journal);
+      await rm(this.claimedFile, { force: true });
+      return;
+    }
+    this.active = true;
+    try {
+      const accepted = {
+        id: request.id, mode: request.mode, requestedAt: request.requestedAt, requestedBy: request.requestedBy ?? null,
+        ...(request.mode === 'cutover' && request.smokeChannel ? { smokeChannel: request.smokeChannel } : {}),
+        acceptedAt: iso(this.now()),
+      };
+      await writeOwnerOnlyJson(this.journalFile, { ...journal, request: accepted, requests: [...(journal.requests || []), accepted.id].slice(-HISTORY) });
+      await rm(this.claimedFile, { force: true });
+      this.log('cutover_request_accepted', { id: accepted.id, mode: accepted.mode });
+      await this.#perform(accepted, accepted.mode);
+    } finally {
+      this.active = false;
+    }
+  }
+
+  async #refuse(request, reason, journal) {
+    const refusal = {
+      id: typeof request?.id === 'string' ? request.id : null, mode: request?.mode ?? null, requestedAt: request?.requestedAt ?? null,
+      reason, at: iso(this.now()),
+    };
+    await writeOwnerOnlyJson(this.journalFile, { ...journal, refusals: [...(journal.refusals || []), refusal].slice(-HISTORY) });
+    this.state = { state: 'refused', requestId: refusal.id, reason };
+    this.log('cutover_request_refused', { id: refusal.id, reason });
+  }
+
+  /**
+   * Runs `action` (cutover or rollback) for an accepted request and records its outcome on the journal. `asOutcome`
+   * is what a rollback that finishes a cut-off cutover reports, rolled back, instead of a plain done; `recovered` names
+   * the boot action that started it, for health.
+   */
+  async #perform(request, action, { asOutcome = null, recovered = null } = {}) {
+    const tag = recovered ? { recovered } : {};
+    this.state = { state: 'running', mode: action, step: null, requestId: request.id, since: iso(this.now()), ...tag };
+    const log = (event, details = {}) => {
+      if (event === 'cutover_step') this.state = { ...this.state, step: details.step };
+      if (event === 'rollback_step') this.state = { ...this.state, phase: 'rollback', step: details.name };
+      this.log(event, details);
+    };
+    // Health says how it ended before the journal does, so whoever sees the outcome sees the same in health.
+    try {
+      const cutover = await this.createCutover({ log });
+      if (action === 'cutover') {
+        await cutover.run({ smokeChannel: request.smokeChannel ?? null, request });
+        this.state = { state: 'serving', step: this.state.step, requestId: request.id, ...tag };
+        await this.#finish(request.id, { state: 'done' });
+        return;
+      }
+      const report = await cutover.rollback({ reason: asOutcome?.code ?? 'asked' });
+      this.state = { state: 'rolled_back', step: this.state.step, requestId: request.id, ...tag };
+      await this.#finish(request.id, asOutcome ? { ...asOutcome, rolledBack: true } : { state: 'done', message: report.message });
+    } catch (error) {
+      const code = error.code || 'HUB_CUTOVER_FAILED';
+      this.state = { state: error.rollback ? 'rolled_back' : 'failed', step: this.state.step, requestId: request.id, code, ...tag };
+      this.log('cutover_request_failed', { id: request.id, code });
+      await this.#finish(request.id, {
+        state: 'failed', code, message: String(error.message).slice(0, 500), rolledBack: Boolean(error.rollback),
+        ...(error.details?.checks ? { checks: error.details.checks } : {}),
+      });
+    }
+  }
+
+  /** Records the request's outcome on the journal, with `extra` journal fields (a boot recovery's). */
+  async #finish(id, outcome, extra = {}) {
+    const journal = await this.#journal();
+    const request = journal.request?.id === id ? journal.request : { id };
+    await writeOwnerOnlyJson(this.journalFile, { ...journal, ...extra, request: { ...request, outcome: { ...outcome, at: iso(this.now()) } } });
+  }
+
+  /** A boot recovery that ends the request without running anything: the outcome and what boot did, in both files. */
+  async #recovered(request, action, outcome, state, extra = {}) {
+    const recovery = { at: iso(this.now()), requestId: request.id, action };
+    this.state = { state, requestId: request.id, recovered: action, ...(outcome.code ? { code: outcome.code } : {}) };
+    this.log('cutover_recovered', { id: request.id, action });
+    await this.#finish(request.id, outcome, { ...extra, recovery });
+  }
+
+  /** On start: a claim a crash left is refused; a request that was running is finished (see the class). */
+  async #recover() {
+    const claimed = await this.#claimed();
+    if (claimed.request || claimed.unreadable) {
+      const journal = await this.#journal();
+      if (!(claimed.request && (journal.requests || []).includes(claimed.request.id))) {
+        await this.#refuse(claimed.request, 'interrupted: the daemon stopped while it was reading this request', journal);
+      }
+      await rm(this.claimedFile, { force: true });
+    }
+    const journal = await this.#journal();
+    const request = journal.request;
+    if (!request?.id) return;
+    if (request.outcome) {
+      this.state = { state: endState(request), requestId: request.id, ...(request.outcome.code ? { code: request.outcome.code } : {}) };
+      return;
+    }
+    const since = request.acceptedAt;
+    const began = Boolean(journal.startedAt) && atOrAfter(journal.startedAt, since);
+    const rolledBack = journal.state === 'rolled_back' && atOrAfter(journal.rolledBackAt, since);
+    const reloading = rollbackProgress(journal, since).some((name) => RELOADING.has(name));
+    const failure = journal.failure ?? { code: 'HUB_CUTOVER_INTERRUPTED', message: 'The Pulse daemon stopped while it ran this cutover.' };
+    // What a cutover that ended rolled back reports; a rollback request that ended so is simply done.
+    const rolledBackOutcome = (message) => (request.mode === 'rollback'
+      ? { state: 'done', message }
+      : { state: 'failed', code: failure.code, message: failure.message, rolledBack: true });
+    this.active = true;
+    try {
+      if (request.mode === 'cutover' && journal.state === 'done' && began) return await this.#recovered(request, 'finished_before_stop', { state: 'done' }, 'serving');
+      if (rolledBack) return await this.#recovered(request, 'rolled_back_before_stop', rolledBackOutcome(journal.rollback?.message ?? null), 'rolled_back');
+      if (request.mode === 'cutover' && !(journal.state === 'running' && began)) {
+        return await this.#recovered(request, 'never_began', {
+          state: 'failed', code: 'HUB_CUTOVER_INTERRUPTED', message: 'The Pulse daemon stopped before this cutover began; nothing was changed.', rolledBack: false,
+        }, 'failed');
+      }
+      if (reloading) return await this.#verifyReloaded(request, journal, rolledBackOutcome);
+      if (request.mode === 'cutover') {
+        const cutFailure = journal.failure ?? {
+          step: STEPS.find((name) => !journal.steps?.[name]) ?? null, code: 'HUB_CUTOVER_INTERRUPTED',
+          message: 'The Pulse daemon stopped while it ran this cutover; it rolled back on start.',
+        };
+        const recovery = { at: iso(this.now()), requestId: request.id, action: 'rolled_back_on_start' };
+        await writeOwnerOnlyJson(this.journalFile, { ...journal, failure: cutFailure, recovery });
+        this.log('cutover_interrupted', { id: request.id, mode: 'cutover', step: cutFailure.step });
+        return await this.#perform(request, 'rollback', { asOutcome: { state: 'failed', code: 'HUB_CUTOVER_INTERRUPTED', message: cutFailure.message }, recovered: 'rolled_back_on_start' });
+      }
+      await writeOwnerOnlyJson(this.journalFile, { ...journal, recovery: { at: iso(this.now()), requestId: request.id, action: 'rollback_run_again' } });
+      this.log('cutover_interrupted', { id: request.id, mode: 'rollback' });
+      return await this.#perform(request, 'rollback', { recovered: 'rollback_run_again' });
+    } finally {
+      this.active = false;
+    }
+  }
+
+  /**
+   * The 20:05:57 DECISION: a rollback that had begun loading the old hub is never run again (it would restore hub.db
+   * under the running hub). Boot only checks that the old hub holds the port, and records rolled_back if it does.
+   */
+  async #verifyReloaded(request, journal, rolledBackOutcome) {
+    this.state = { state: 'running', mode: 'verify', step: 'hub_loading', requestId: request.id, since: iso(this.now()) };
+    let onPort;
+    try {
+      onPort = await (await this.createCutover({ log: this.log })).verifyRolledBack();
+    } catch (error) {
+      onPort = { ok: false, detail: `the check failed: ${error.code || 'error'}: ${String(error.message).slice(0, 200)}` };
+    }
+    if (onPort.ok) {
+      const at = iso(this.now());
+      const message = `Rolled back to the local hub: the Pulse daemon restarted after the old hub was loaded again, and found it on the port (${onPort.detail}).`;
+      return this.#recovered(request, 'verified_old_hub', rolledBackOutcome(message), 'rolled_back', {
+        state: 'rolled_back', rolledBackAt: at, rollback: { ...journal.rollback, recoveredOnBoot: true, verified: onPort.detail, message },
+      });
+    }
+    return this.#recovered(request, 'old_hub_not_serving', {
+      state: 'failed', code: 'HUB_ROLLBACK_INTERRUPTED', rolledBack: false,
+      message: `The Pulse daemon stopped after its rollback began loading the old hub, and the old hub does not hold the port now (${onPort.detail}). It is not rolled back twice, which would restore hub.db under a hub that may have it open: check the hub by hand.`,
+    }, 'failed');
+  }
+}
+
+/** Whether the daemon behind this health file is alive and serves cutover.request. */
+export function takesCutoverRequests(health, now = Date.now()) {
+  const ageMs = now - Date.parse(health?.heartbeatAt);
+  return ageMs <= HEALTH_FRESH_MS && Number(health?.cutoverProtocol) >= CUTOVER_PROTOCOL;
+}
+
+/** Creates `file` owner-only with `value`, whole or not at all, and only if it does not exist yet. */
+async function createExclusive(file, value) {
+  const temporary = path.join(path.dirname(file), `.${path.basename(file)}.${process.pid}.${randomBytes(6).toString('hex')}.tmp`);
+  const handle = await open(temporary, 'wx', 0o600);
+  try {
+    await handle.writeFile(`${JSON.stringify(value, null, 2)}\n`, 'utf8');
+    await handle.sync();
+  } finally {
+    await handle.close();
+  }
+  try {
+    await chmod(temporary, 0o600);
+    await link(temporary, file);
+    return true;
+  } catch (error) {
+    if (error.code === 'EEXIST') return false;
+    throw error;
+  } finally {
+    await rm(temporary, { force: true });
+  }
+}
+
+/** Writes an owner-only cutover.request for the daemon, exclusively: one request at a time. */
+export async function writeCutoverRequest({ configDir, mode, smokeChannel = null, health = null, now = Date.now, id = randomUUID(), requestedBy = null }) {
+  const pending = (detail) => new HubError('HUB_CUTOVER_PENDING', `Another cutover request is ${detail}; wait for it to finish (see ${path.join(configDir, CUTOVER_FILE)}).`);
+  if (health?.cutover?.state === 'running') throw pending(`running in the Pulse daemon (${health.cutover.requestId}${health.cutover.step ? `, at ${health.cutover.step}` : ''})`);
+  const request = {
+    id, mode, requestedAt: iso(now()),
+    requestedBy: requestedBy ?? { pid: process.pid, session: process.env.CURSOR_CONVERSATION_ID || null },
+    ...(smokeChannel ? { smokeChannel } : {}),
+  };
+  if (!(await createExclusive(path.join(configDir, CUTOVER_REQUEST_FILE), request))) throw pending('waiting for the Pulse daemon');
+  return request;
+}
+
+/**
+ * Follows a request through cutover.json, printing each step as `log(event, details)` exactly as an in-process run
+ * logs it. It resolves with the journal (a cutover) or the rollback report, or throws: the journal's code for a run
+ * that failed (or rolled back), HUB_CUTOVER_REFUSED, or HUB_CUTOVER_NOT_ACCEPTED when nobody took it within
+ * `acceptMs` (it is withdrawn). It ends without an outcome, with `exit` FOLLOW_UNKNOWN_EXIT, when the daemon's heartbeat
+ * goes stale (it finishes the request when it starts again) or after `timeoutMs`, the worst case of a run.
+ */
+export async function followCutoverRequest({
+  configDir, request, log = () => {}, now = Date.now, wait = sleep, pollMs = POLL_MS, acceptMs = ACCEPT_MS, timeoutMs = FOLLOW_MS,
+  healthFreshMs = HEALTH_FRESH_MS,
+}) {
+  const journalFile = path.join(configDir, CUTOVER_FILE);
+  const requestFile = path.join(configDir, CUTOVER_REQUEST_FILE);
+  const healthFile = path.join(configDir, 'daemon-health.json');
+  const started = now();
+  const printed = { steps: new Set(), failure: false, rollback: 0 };
+  let withDaemon = false;
+  const print = (journal) => {
+    const since = journal.request.acceptedAt;
+    if (request.mode === 'cutover' && journal.startedAt && atOrAfter(journal.startedAt, since)) {
+      for (const name of STEPS) {
+        if (!journal.steps?.[name] || printed.steps.has(name)) continue;
+        printed.steps.add(name);
+        const { at: _at, ...detail } = journal.steps[name];
+        log('cutover_step', { step: name, ...detail });
+      }
+      if (journal.failure && !printed.failure) {
+        printed.failure = true;
+        log('cutover_failed', journal.failure);
+      }
+    }
+    if (rollbackProgress(journal, since).length) {
+      for (const entry of journal.rollback.steps.slice(printed.rollback)) log('rollback_step', entry);
+      printed.rollback = journal.rollback.steps.length;
+    }
+  };
+  const unknown = (code, message) => new HubError(code, message, { exit: FOLLOW_UNKNOWN_EXIT });
+  for (;;) {
+    const journal = await readOwnerOnlyJson(journalFile).catch(() => null);
+    const refused = journal?.refusals?.find((entry) => entry.id === request.id);
+    if (refused) throw new HubError('HUB_CUTOVER_REFUSED', `The Pulse daemon refused request ${request.id}: ${refused.reason}`, { details: { refused } });
+    const health = await readOwnerOnlyJson(healthFile).catch(() => null);
+    // A long synchronous stretch (a rescan) can delay a heartbeat; a daemon whose pid is alive gets three windows.
+    const ageMs = now() - Date.parse(health?.heartbeatAt);
+    const stale = !(ageMs <= healthFreshMs) && (!(ageMs <= 3 * healthFreshMs) || !(Number.isInteger(health?.pid) && pidAlive(health.pid)));
+    if (journal?.request?.id === request.id) {
+      print(journal);
+      const outcome = journal.request.outcome;
+      if (outcome?.state === 'done') return request.mode === 'cutover' ? journal : journal.rollback;
+      if (outcome) {
+        throw Object.assign(
+          new HubError(outcome.code || 'HUB_CUTOVER_FAILED', outcome.message || 'the cutover failed', { details: { ...(outcome.checks ? { checks: outcome.checks } : {}), rolledBack: Boolean(outcome.rolledBack) } }),
+          outcome.rolledBack ? { rollback: journal.rollback ?? null } : {},
+        );
+      }
+      if (stale) {
+        throw unknown('HUB_CUTOVER_DAEMON_STOPPED', `The Pulse daemon stopped (last heartbeat ${health?.heartbeatAt ?? 'never'}) while it ran request ${request.id}. When it starts again it finishes the request (a cutover that had begun rolls back); follow it in ${journalFile}.`);
+      }
+    } else if (!withDaemon && (stale || now() - started >= acceptMs)) {
+      // Withdrawn only while it is still ours on the disk: gone means the daemon claimed it, so it is followed.
+      const pending = await readOwnerOnlyJson(requestFile).catch(() => null);
+      let withdrawn = false;
+      if (pending?.id === request.id) {
+        try {
+          await unlink(requestFile);
+          withdrawn = true;
+        } catch (error) {
+          if (error.code !== 'ENOENT') throw error;
+        }
+      }
+      if (withdrawn) {
+        throw new HubError('HUB_CUTOVER_NOT_ACCEPTED', stale
+          ? `The Pulse daemon is not running (last heartbeat ${health?.heartbeatAt ?? 'never'}), so request ${request.id} was withdrawn. Start it with \`agentpulse pulse start\`.`
+          : `The Pulse daemon did not take request ${request.id} within ${Math.round(acceptMs / 1000)} s, so it was withdrawn. Check it with \`agentpulse pulse status\`.`);
+      }
+      withDaemon = true;
+    } else if (withDaemon && stale) {
+      throw unknown('HUB_CUTOVER_DAEMON_STOPPED', `The Pulse daemon stopped (last heartbeat ${health?.heartbeatAt ?? 'never'}) after it claimed request ${request.id}; when it starts again it refuses or finishes it. Follow it in ${journalFile}.`);
+    }
+    if (now() - started >= timeoutMs) {
+      throw unknown('HUB_CUTOVER_FOLLOW_TIMEOUT', `Request ${request.id} is still running in the Pulse daemon after ${Math.round(timeoutMs / 60_000)} min; follow it in ${journalFile} or with \`agentpulse pulse status\`.`);
+    }
+    await wait(pollMs);
+  }
+}

package/src/lib/cutoverWiring.js

@@ -0,0 +1,140 @@
+import { randomBytes } from 'node:crypto';
+import { readFile } from 'node:fs/promises';
+import path from 'node:path';
+import { fileURLToPath } from 'node:url';
+import { CloudFirstCommands } from './cloudFirstCommands.js';
+import { Cutover } from './cutover.js';
+import { HubError } from './errors.js';
+import { discoverLocalHub } from './localHub.js';
+import { ServiceManager } from './serviceManager.js';
+import { defaultSecretStore } from './secretStore.js';
+import { backupDatabase } from './sqlite.js';
+import { readOwnerOnlyJson } from './storage.js';
+import { execFileBounded, spawnBounded } from './subprocess.js';
+import { TokenStore } from './tokenStore.js';
+
+const RUNNER = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..', 'runner');
+const LSOF = '/usr/sbin/lsof';
+
+async function hubToken(hubHome) {
+  return (await readFile(path.join(hubHome, 'state', 'hub.token'), 'utf8')).trim();
+}
+
+async function local(port, route, token, { method = 'GET', body = null, timeoutMs = 3_000 } = {}) {
+  const res = await fetch(`http://127.0.0.1:${port}${route}`, {
+    method, signal: AbortSignal.timeout(timeoutMs),
+    headers: { authorization: `Bearer ${token}`, ...(body ? { 'content-type': 'application/json' } : {}) },
+    body: body ? JSON.stringify(body) : undefined,
+  });
+  return { status: res.status, body: await res.json().catch(() => null) };
+}
+
+/** Who has these files open, by `lsof -t`. Exit 1 with no output means nobody; anything else unexpected is unknown. */
+export async function lsofHolders(files, { run = execFileBounded } = {}) {
+  try {
+    const { stdout } = await run(LSOF, ['-t', ...files], { timeout: 10_000 });
+    return { pids: String(stdout).split(/\s+/).filter(Boolean).map(Number).filter(Number.isInteger) };
+  } catch (error) {
+    if (error.code === 1 && !String(error.stdout || '').trim()) return { pids: [] };
+    return { error: String(error.message || error).slice(0, 200) };
+  }
+}
+
+/**
+ * Who owns the LISTEN socket on 127.0.0.1:port, each with its parent, by `lsof -FpR` (ps is not allowed in an agent's
+ * sandbox; lsof is). Exit 1 with no output means nobody; anything else unexpected is unknown.
+ */
+export async function lsofPortOwners(port, { run = execFileBounded } = {}) {
+  try {
+    const { stdout } = await run(LSOF, ['-nP', `-iTCP@127.0.0.1:${Number(port)}`, '-sTCP:LISTEN', '-FpR'], { timeout: 10_000 });
+    const owners = [];
+    for (const line of String(stdout).split('\n')) {
+      const value = Number(line.slice(1));
+      if (!Number.isInteger(value)) continue;
+      if (line.startsWith('p') && !owners.some((o) => o.pid === value)) owners.push({ pid: value, ppid: null });
+      else if (line.startsWith('R') && owners.length) owners.at(-1).ppid ??= value;
+    }
+    return { owners };
+  } catch (error) {
+    if (error.code === 1 && !String(error.stdout || '').trim()) return { owners: [] };
+    return { error: String(error.message || error).slice(0, 200) };
+  }
+}
+
+/**
+ * After the switch: `bus ask` to the manager through the installed shim reaches the runner, a hook delivers a probe
+ * message, and (with a channel) one post lands through the facade. Nothing here talks to hub.db.
+ */
+export async function smokeTest({ hubHome, port, channel = null, run = spawnBounded, node = process.execPath }) {
+  const token = await hubToken(hubHome);
+  const probe = `cutover-probe-${randomBytes(4).toString('hex')}`;
+  const cli = (...argv) => run(node, [path.join(hubHome, 'hub.mjs'), ...argv], { timeoutMs: 15_000 }).then((r) => JSON.parse(r.stdout));
+  const checks = [];
+  const ask = await cli('bus', 'ask', '--as', probe, '--topic', 'manager', 'cutover smoke test: no answer needed');
+  if (ask.spooled || !ask.to?.includes('manager')) return { ok: false, reason: `bus ask did not reach the runner: ${JSON.stringify(ask).slice(0, 200)}` };
+  checks.push('ask');
+  await cli('bus', 'register', '--as', probe, '--role', 'CHECK');
+  // The bus refuses mail to an address it has never seen, so the probe's first hook makes it a session; the second shows the mail.
+  const hook = () => run(node, [path.join(hubHome, 'hooks', 'hub-bridge.mjs')], {
+    timeoutMs: 5_000, input: JSON.stringify({ hook_event_name: 'postToolUse', conversation_id: probe, tool_name: 'Read' }),
+  });
+  await hook();
+  const sent = await cli('bus', 'publish', '--as', 'user', '--topic', `agent/${probe}`, 'cutover hook check');
+  if (!sent.id) return { ok: false, reason: `the probe message was refused: ${JSON.stringify(sent).slice(0, 200)}` };
+  if (!String((await hook()).stdout).includes(sent.id)) return { ok: false, reason: 'the hook did not deliver the probe message' };
+  checks.push('hook');
+  if (channel) {
+    const posted = await local(port, '/api/chat/post', token, { method: 'POST', body: { ch: `task:${channel}`, text: 'Cutover smoke test: Pulse now serves this hub.' } });
+    if (posted.status !== 200) return { ok: false, reason: `the channel post failed: ${posted.status}` };
+    checks.push('channel');
+  }
+  return { ok: true, checks, probe };
+}
+
+/**
+ * Who answers on the port: the Pulse facade (it has /runner/health) or the old hub (it doesn't, and serves the API).
+ * `timeoutMs` bounds each request; rollback waits longer for a hub that is busy right after it loads.
+ */
+export function portProbes(tokenFor) {
+  return {
+    facadeUp: async (p) => {
+      try { return (await local(p, '/runner/health', await tokenFor())).status === 200; } catch { return false; }
+    },
+    oldHubUp: async (p, { timeoutMs = 3_000 } = {}) => {
+      try {
+        const token = await tokenFor();
+        return (await local(p, '/runner/health', token, { timeoutMs })).status === 404 && (await local(p, '/api/settings', token, { timeoutMs })).status === 200;
+      } catch { return false; }
+    },
+  };
+}
+
+/**
+ * The production seams of `pulse cutover`: launchd, lsof, the Phase 2 import and reconcile, and the local port.
+ * `serve` is the daemon's own runner switch when the daemon runs it (CUT-03); in-process it is left to the heartbeat.
+ */
+export async function createCutover({ configDir, log = () => {}, port = 8890, serve = undefined }) {
+  const config = await readOwnerOnlyJson(path.join(configDir, 'config.json'));
+  if (!config) throw new HubError('HUB_NOT_INSTALLED', 'Pulse is not installed here; run `agentpulse pulse install`');
+  const hubHome = config.cloudFirst?.hubHome || config.mirror?.hub?.dir || (await discoverLocalHub({ config })).dir;
+  const tokenFor = () => hubToken(hubHome);
+  return new Cutover({
+    configDir, hubHome, port, log, serve,
+    services: new ServiceManager({}),
+    probe: { holders: (files) => lsofHolders(files), portOwners: (p) => lsofPortOwners(p) },
+    cloudFirst: new CloudFirstCommands({ configDir }),
+    daemon: portProbes(tokenFor),
+    smoke: (options) => smokeTest(options),
+    backupDb: backupDatabase,
+    applySpool: (file) => spoolIntoHubDb(file, { hubHome }),
+    signIn: new TokenStore({ secretStore: defaultSecretStore(), configDir }),
+  });
+}
+
+/** Rollback's spool step: the freeze's spooled writes applied to the restored hub.db, in a child running the hub core. */
+export async function spoolIntoHubDb(file, { hubHome, env = {} }) {
+  const { stdout } = await execFileBounded(process.execPath, [path.join(RUNNER, 'spoolInto.js'), file], {
+    timeout: 120_000, env: { ...process.env, ...env, PULSE_HUB_HOME: hubHome },
+  });
+  return JSON.parse(stdout);
+}

package/src/lib/daemonLock.js

@@ -1,6 +1,7 @@
 import path from 'node:path';
 import { HubError } from './errors.js';
 import { acquireOwnerLock } from './ownerLock.js';
+import { CONNECTOR_VERSION } from './pairingLifecycle.js';
 import { ProcessIdentityProvider } from './processOwnership.js';
 import { ensureOwnerOnlyDirectory, writeOwnerOnlyJson } from './storage.js';
 
@@ -41,6 +42,7 @@ export class DaemonLock {
       pid: this.pid,
       heartbeatAt: new Date().toISOString(),
       ...status,
+      connectorVersion: CONNECTOR_VERSION,
     });
   }
 

package/src/lib/daemonLoop.js

@@ -10,7 +10,7 @@ export const MIRROR_UPDATE_STEP = 'mirror_update';
  * adds fields such as the restart state to every health write.
  */
 export function createHeartbeat({
-  readConfig, runtime, mirror, lock, ledger, ownerStore, progress = () => {}, extraHealth = () => ({}),
+  readConfig, runtime, mirror, shadow = null, runner = null, lock, ledger, ownerStore, progress = () => {}, extraHealth = () => ({}),
 }) {
   let last = null;
   const health = (config, status) => ({
@@ -56,7 +56,10 @@ export function createHeartbeat({
     }
     progress(MIRROR_UPDATE_STEP);
     // A cloud that refused the contract would refuse every batch too, so the mirror waits for the next session.
-    await mirror.update(TERMINAL_STATES.has(status.state) || status.code === 'contract_unsupported' ? null : config);
+    const usable = !(TERMINAL_STATES.has(status.state) || status.code === 'contract_unsupported');
+    await mirror.update(usable ? config : null);
+    shadow?.update(usable ? config : null);
+    runner?.update(usable ? config : null);
     progress('health');
     last = { config, status };
     await write();