@agentpulselabs/cli-plugin-pulse 1.0.0 → 1.1.2

package/src/lib/cutover.js

@@ -0,0 +1,376 @@
+import { createHash } from 'node:crypto';
+import { chmod, copyFile, readFile, rm, stat, writeFile } from 'node:fs/promises';
+import path from 'node:path';
+import { setTimeout as sleep } from 'node:timers/promises';
+import { clientState, installClients, restoreClients } from './clientInstaller.js';
+import { HubError } from './errors.js';
+import { RunnerStore, SEED_META } from './runnerStore.js';
+import { readOwnerOnlyJson, writeOwnerOnlyJson } from './storage.js';
+
+export const CUTOVER_FILE = 'cutover.json';
+/** The oldest daemon that finds the sign-in under its per-config Keychain name, which step 0 moves it to (M1). */
+export const MIN_DAEMON_VERSION = '1.1.0';
+const HEALTH_FRESH_MS = 60_000;
+export const OLD_HUB_LABELS = Object.freeze(['com.agentpulse.agents-hub', 'com.agentpulse.keepawake']);
+/** Rollback's wait for the restored hub: a hub busy with discovery right after it loads answers slowly. */
+export const ROLLBACK_PROBE_MS = 10_000;
+export const BACKUP_SUFFIX = '.pre-cutover';
+/** The steps of a cutover (ADR-4 and the architect's 10:44 order), in order. */
+export const STEPS = Object.freeze(['sign_in_moved', 'shims_offline', 'hub_unloaded', 'hub_db_quiet', 'final_delta', 'reconciled', 'backed_up', 'daemon_serving', 'shims_online', 'smoke']);
+
+const sha256File = async (file) => createHash('sha256').update(await readFile(file)).digest('hex');
+
+/** `version >= floor` on the numeric x.y.z core; anything unparsable is older. */
+function atLeast(version, floor) {
+  const parse = (v) => String(v ?? '').match(/^(\d+)\.(\d+)\.(\d+)/)?.slice(1).map(Number) ?? null;
+  const [have, need] = [parse(version), parse(floor)];
+  if (!have) return false;
+  for (let i = 0; i < 3; i += 1) if (have[i] !== need[i]) return have[i] > need[i];
+  return true;
+}
+
+/**
+ * `agentpulse pulse cutover`: the local hub's work moves into the Pulse daemon on this machine, in a short freeze, with
+ * a backup kept; `--rollback` restores that backup and the old hub. Every effect goes through an injected seam:
+ * `services` (launchd, by label), `probe` (who holds hub.db), `cloudFirst` (Phase 2's final import and reconcile),
+ * `daemon` (the facade's health and the old hub's), `smoke`, `backupDb`, `applySpool` and `signIn` (the TokenStore). Each step is journaled in
+ * cutover.json before the next starts; a failure at any step after the freeze began rolls back by itself.
+ */
+export class Cutover {
+  constructor({
+    configDir, hubHome, port = 8890, services, probe, cloudFirst, daemon, smoke, backupDb, applySpool, signIn,
+    labels = OLD_HUB_LABELS, holderWaitMs = 120_000, facadeWaitMs = 90_000, pollMs = 1_000, now = Date.now, wait = sleep, log = () => {},
+  }) {
+    this.configDir = configDir;
+    this.configFile = path.join(configDir, 'config.json');
+    this.journalFile = path.join(configDir, CUTOVER_FILE);
+    this.hubHome = hubHome;
+    this.hubDb = path.join(hubHome, 'state', 'hub.db');
+    this.backupFile = `${this.hubDb}${BACKUP_SUFFIX}`;
+    Object.assign(this, { port, services, probe, cloudFirst, daemon, smoke, backupDb, applySpool, signIn, labels, holderWaitMs, facadeWaitMs, pollMs, now, wait, log });
+  }
+
+  async journal() {
+    return (await readOwnerOnlyJson(this.journalFile)) || { state: 'none', steps: {} };
+  }
+
+  async #record(journal, step, detail = {}) {
+    journal.steps[step] = { at: new Date(this.now()).toISOString(), ...detail };
+    journal.updatedAt = journal.steps[step].at;
+    await writeOwnerOnlyJson(this.journalFile, journal);
+    this.log('cutover_step', { step, ...detail });
+  }
+
+  async #config() {
+    const config = await readOwnerOnlyJson(this.configFile);
+    if (!config) throw new HubError('HUB_NOT_INSTALLED', 'Pulse is not installed here; run `agentpulse pulse install`');
+    return config;
+  }
+
+  /** Who has hub.db or its WAL open, besides this process. Unknown (lsof failed) is never "nobody" (N1). */
+  async #holders() {
+    const found = await this.probe.holders([this.hubDb, `${this.hubDb}-wal`]);
+    if (found.error) return { known: false, pids: [], error: found.error };
+    return { known: true, pids: found.pids.filter((pid) => pid !== process.pid) };
+  }
+
+  /**
+   * CUT-01: the old hub holds the port. It answers HTTP, or it is hung: its launchd job is loaded, the job's pid (or a
+   * child of it, by lsof's parent pid) owns the LISTEN socket, and the facade does not answer there. The cutover replaces a hung hub anyway,
+   * so being healthy is not required. It fails when the port is free, another process owns it, or the facade serves it.
+   */
+  async #oldHubOnPort() {
+    const where = `127.0.0.1:${this.port}`;
+    if (await this.daemon.oldHubUp(this.port)) return { ok: true, detail: `answers HTTP on ${where}` };
+    if (await this.daemon.facadeUp(this.port)) return { ok: false, detail: `the Pulse facade already serves ${where}` };
+    const found = await this.probe.portOwners(this.port);
+    if (found.error) return { ok: false, detail: `cannot tell who holds ${where} (${found.error})` };
+    if (!found.owners.length) return { ok: false, detail: `${where} is free: the old hub is not running` };
+    const pids = found.owners.map((o) => o.pid).join(', ');
+    const [label] = this.labels;
+    const job = await this.services.labelState(label);
+    if (!job.loaded || !job.pid) return { ok: false, detail: `${where} is held by pid ${pids}, but ${label} is not running` };
+    const owner = found.owners.find((o) => o.pid === job.pid || o.ppid === job.pid);
+    if (!owner) return { ok: false, detail: `${where} is held by pid ${pids}, not by ${label} (pid ${job.pid}) or a child of it` };
+    return { ok: true, hung: true, pid: owner.pid, detail: `held by pid ${owner.pid}, not answering HTTP (it is replaced anyway)` };
+  }
+
+  /** The cloud seq this machine's import reached: everything at or below it is already in the seed (N3). */
+  #cutoverSeq() {
+    const store = RunnerStore.openReadOnly(this.configDir);
+    if (!store) return 0;
+    try {
+      return Number(store.db.prepare('SELECT MAX(CAST(seq AS INTEGER)) AS seq FROM sent').get()?.seq || 0);
+    } finally {
+      store.close();
+    }
+  }
+
+  /** The cutover backup runner.db's hub tables were seeded from, or null. */
+  #seed() {
+    const store = RunnerStore.openReadOnly(this.configDir);
+    if (!store) return null;
+    try {
+      return store.db.prepare("SELECT 1 AS found FROM sqlite_master WHERE type = 'table' AND name = 'meta'").get() ? store.meta(SEED_META) : null;
+    } finally {
+      store.close();
+    }
+  }
+
+  /** M1: the running daemon, by its last heartbeat, runs connector code that reads the sign-in where step 0 puts it. */
+  async #daemonVersion() {
+    const health = await readOwnerOnlyJson(path.join(this.configDir, 'daemon-health.json')).catch(() => null);
+    if (!health) return { ok: false, detail: 'no daemon-health.json: the Pulse daemon is not running' };
+    const ageMs = this.now() - Date.parse(health.heartbeatAt);
+    if (!(ageMs <= HEALTH_FRESH_MS)) return { ok: false, detail: `the daemon's last heartbeat was ${Number.isFinite(ageMs) ? `${Math.round(ageMs / 1000)} s` : 'never'} ago` };
+    if (!health.connectorVersion) return { ok: false, detail: `pid ${health.pid} reports no connector version: it runs code from before ${MIN_DAEMON_VERSION}` };
+    return { ok: atLeast(health.connectorVersion, MIN_DAEMON_VERSION), detail: `connector ${health.connectorVersion}, pid ${health.pid}` };
+  }
+
+  /** B1: a seed is replaced only after a rollback listed what the runner wrote on top of it. */
+  #seedCheck() {
+    const seed = this.#seed();
+    if (!seed) return { ok: true, detail: 'not seeded yet: this cutover seeds it from the new backup' };
+    const from = `${seed.sha256 ? `backup ${seed.sha256.slice(0, 12)}` : 'a backup with no sha256 recorded'} (${seed.file})`;
+    if (seed.rolledBackAt) return { ok: true, detail: `seeded from ${from}, rolled back at ${seed.rolledBackAt}: this cutover reseeds it from the new backup` };
+    return { ok: false, detail: `seeded from ${from} and never rolled back: reseeding would drop what the runner wrote since, unlisted` };
+  }
+
+  /** Everything the cutover checks before it touches anything, and what it would change. Reads only. */
+  async dryRun() {
+    const checks = [];
+    const check = (name, ok, detail = null, fix = null) => checks.push({ name, ok: Boolean(ok), ...(detail ? { detail } : {}), ...(fix ? { fix } : {}) });
+    const config = await this.#config();
+    const capabilities = config.capabilities || [];
+    check('paired', Boolean(config.paired?.machineId), config.paired?.machineId || 'not paired', 'agentpulse pulse pair');
+    check('grant reads and writes the log', ['hub.sync', 'log.write'].every((c) => capabilities.includes(c)), capabilities.join(', '), 'agentpulse pulse log-write on');
+    check('not already cut over', config.cloudFirst?.mode !== 'runner', config.cloudFirst?.mode || 'off', 'agentpulse pulse cutover --rollback');
+    const daemon = await this.#daemonVersion();
+    check(`the daemon runs connector ${MIN_DAEMON_VERSION} or later`, daemon.ok, daemon.detail, `link the Pulse ${MIN_DAEMON_VERSION} plugin, then agentpulse pulse restart`);
+    const seed = this.#seedCheck();
+    check('runner.db seed', seed.ok, seed.detail, 'agentpulse pulse cutover --rollback');
+    let hubDbSize = 0;
+    try { hubDbSize = (await stat(this.hubDb)).size; check('hub.db', true, `${hubDbSize} bytes`); } catch { check('hub.db', false, this.hubDb); }
+    try {
+      const token = await stat(path.join(this.hubHome, 'state', 'hub.token'));
+      check('hub token is owner-only', (token.mode & 0o077) === 0, (token.mode & 0o777).toString(8));
+    } catch { check('hub token', false, 'missing'); }
+    const entry = await clientState({ hubHome: this.hubHome });
+    check('entry points are agents-hub 6b2fb0b', Object.values(entry).every((s) => s === 'original'), JSON.stringify(entry));
+    for (const label of this.labels) {
+      const state = await this.services.labelState(label);
+      check(`${label} installed`, state.installed, state.loaded ? `loaded, pid ${state.pid}` : 'not loaded');
+    }
+    const onPort = await this.#oldHubOnPort();
+    check('the old hub holds the port', onPort.ok, onPort.detail, `start the old hub, or see what holds the port: lsof -nP -iTCP@127.0.0.1:${this.port} -sTCP:LISTEN`);
+    const holders = await this.#holders();
+    check('who holds hub.db now (informative)', true, holders.known ? `${holders.pids.length} process(es): ${holders.pids.join(', ') || 'none'}` : `unknown: ${holders.error}`);
+    check('no backup left from an earlier cutover', !(await stat(this.backupFile).catch(() => null)), this.backupFile, 'rename or remove it first');
+    const ok = checks.every((c) => c.ok);
+    return {
+      command: 'cutover',
+      dryRun: true,
+      ok,
+      checks,
+      wouldChange: {
+        files: Object.keys(entry).map((name) => ({ name, becomes: 'Pulse thin client', keptAs: `<file>${BACKUP_SUFFIX}` })),
+        launchAgents: this.labels.map((label) => ({ label, becomes: 'unloaded and disabled' })),
+        backup: this.backupFile,
+        config: { 'cloudFirst.mode': 'runner', 'cloudFirst.hubHome': this.hubHome, 'cloudFirst.port': this.port, 'mirror.hub': { dir: this.hubHome, port: this.port } },
+        cutoverSeq: this.#cutoverSeq(),
+        signIn: await this.signIn.secretPlan(),
+      },
+      outage: {
+        what: 'from shims offline (step 1) to shims online (step 8): hooks answer {}, bus listen sees no new mail, writes spool under printed ids',
+        boundMs: this.holderWaitMs + 2 * 10 * 60_000 + this.facadeWaitMs,
+      },
+      message: ok ? 'Ready to cut over. Nothing was changed.' : 'Not ready: see the checks that failed. Nothing was changed.',
+    };
+  }
+
+  /** The cutover. Resolves with the journal; a failure after the freeze began rolls back and rethrows. */
+  async run({ smokeChannel = null } = {}) {
+    const preflight = await this.dryRun();
+    if (!preflight.ok) throw new HubError('HUB_CUTOVER_NOT_READY', preflight.message, { details: { checks: preflight.checks.filter((c) => !c.ok) } });
+    const journal = { state: 'running', startedAt: new Date(this.now()).toISOString(), hubHome: this.hubHome, port: this.port, steps: {}, loadedBefore: [] };
+    await writeOwnerOnlyJson(this.journalFile, journal);
+    try {
+      // Before the freeze: the sign-in moves to this config's own Keychain item (the 12:52 DECISION); no outage.
+      const moved = await this.signIn.migrateSecret();
+      await this.#record(journal, 'sign_in_moved', moved.moved ? { from: moved.from, to: moved.to } : { to: moved.reference });
+      await installClients({ hubHome: this.hubHome, port: this.port, offline: true });
+      await this.#record(journal, 'shims_offline');
+      for (const label of this.labels) {
+        const before = await this.services.labelState(label);
+        if (before.loaded) journal.loadedBefore.push(label);
+        await this.services.retireLabel(label);
+      }
+      await this.#record(journal, 'hub_unloaded', { labels: this.labels });
+      const deadline = this.now() + this.holderWaitMs;
+      let holders = await this.#holders();
+      while (!(holders.known && holders.pids.length === 0)) {
+        if (this.now() >= deadline) {
+          throw new HubError('HUB_CUTOVER_HOLDERS', holders.known
+            ? `hub.db is still open by ${holders.pids.join(', ')} after ${this.holderWaitMs} ms`
+            : `cannot tell who holds hub.db (${holders.error})`);
+        }
+        await this.wait(this.pollMs);
+        holders = await this.#holders();
+      }
+      await this.#record(journal, 'hub_db_quiet');
+      const delta = await this.cloudFirst.import({ wait: true });
+      await this.#record(journal, 'final_delta', { message: delta?.message ?? null });
+      // CUT-02: channels whose file is gone (retired) and text Pulse holds more redacted (redacted) are listed, not lost.
+      const reconcile = await this.cloudFirst.reconcile();
+      const outcome = { classes: reconcile.classes ?? null, retired: reconcile.retired ?? [], redacted: reconcile.redacted ?? [] };
+      if (reconcile.exitCode !== 0) {
+        journal.reconcile = outcome;
+        throw new HubError('HUB_CUTOVER_RECONCILE', `reconcile is not all equal: ${reconcile.message || reconcile.status}`);
+      }
+      await this.#record(journal, 'reconciled', { message: reconcile.message ?? null, ...outcome });
+      await this.backupDb(this.hubDb, this.backupFile);
+      await chmod(this.backupFile, 0o444);
+      journal.backup = { file: this.backupFile, sha256: await sha256File(this.backupFile) };
+      await this.#record(journal, 'backed_up', journal.backup);
+      const cutoverSeq = this.#cutoverSeq();
+      const config = await this.#config();
+      await writeOwnerOnlyJson(this.configFile, {
+        ...config,
+        mirror: { ...(config.mirror || {}), hub: { dir: this.hubHome, port: this.port } },
+        cloudFirst: { ...(config.cloudFirst || {}), mode: 'runner', hubHome: this.hubHome, port: this.port, cutoverSeq, seedFrom: this.backupFile, seedSha256: journal.backup.sha256, shadow: false },
+      });
+      journal.cutoverSeq = cutoverSeq;
+      const serving = this.now() + this.facadeWaitMs;
+      while (!(await this.daemon.facadeUp(this.port))) {
+        if (this.now() >= serving) throw new HubError('HUB_CUTOVER_FACADE', `the Pulse daemon did not serve 127.0.0.1:${this.port} within ${this.facadeWaitMs} ms`);
+        await this.wait(this.pollMs);
+      }
+      journal.cutoverAt = this.now();
+      await this.#record(journal, 'daemon_serving', { cutoverSeq });
+      await installClients({ hubHome: this.hubHome, port: this.port, offline: false });
+      await this.#record(journal, 'shims_online');
+      const smoke = await this.smoke({ hubHome: this.hubHome, port: this.port, channel: smokeChannel });
+      if (!smoke.ok) throw new HubError('HUB_CUTOVER_SMOKE', `smoke test failed: ${smoke.reason}`);
+      await this.#record(journal, 'smoke', smoke);
+      journal.state = 'done';
+      await writeOwnerOnlyJson(this.journalFile, journal);
+      return journal;
+    } catch (error) {
+      journal.failure = { step: STEPS.find((s) => !journal.steps[s]) ?? null, code: error.code || 'error', message: String(error.message).slice(0, 300) };
+      await writeOwnerOnlyJson(this.journalFile, journal);
+      this.log('cutover_failed', journal.failure);
+      const rolled = await this.rollback({ reason: journal.failure.code });
+      throw Object.assign(error, { rollback: rolled });
+    }
+  }
+
+  /**
+   * Back to the local hub from any point (ADR-4): the daemon leaves the port, what was written since the switch is
+   * exported, the entry points are restored byte for byte, hub.db comes back from the backup, the freeze's spool is
+   * applied to it by edge id, and the old LaunchAgents load again. Loses only what was written after the switch.
+   */
+  async rollback({ reason = 'asked' } = {}) {
+    const journal = await this.journal();
+    const report = { command: 'cutover', rollback: true, reason, steps: [] };
+    const step = (name, detail = {}) => { report.steps.push({ name, ...detail }); this.log('rollback_step', { name, ...detail }); };
+    const config = await this.#config();
+    if (config.cloudFirst?.mode === 'runner') {
+      await writeOwnerOnlyJson(this.configFile, { ...config, cloudFirst: { ...config.cloudFirst, mode: 'rolled_back' } });
+      const deadline = this.now() + this.facadeWaitMs;
+      while (await this.daemon.facadeUp(this.port)) {
+        if (this.now() >= deadline) throw new HubError('HUB_ROLLBACK_FACADE', `the Pulse daemon still serves 127.0.0.1:${this.port}`);
+        await this.wait(this.pollMs);
+      }
+      step('daemon_left_port');
+    }
+    if (journal.cutoverAt) {
+      const lost = await this.#exportLost(journal.cutoverAt);
+      report.lost = lost;
+      step('lost_exported', lost);
+    }
+    const marked = await this.#markSeedRolledBack();
+    if (marked) step('runner_db_marked', marked);
+    await restoreClients({ hubHome: this.hubHome });
+    step('entry_points_restored');
+    if (journal.steps?.backed_up && (await stat(this.backupFile).catch(() => null))) {
+      for (const suffix of ['-wal', '-shm']) await rm(`${this.hubDb}${suffix}`, { force: true });
+      const restored = `${this.hubDb}.pulse-restore`;
+      await copyFile(this.backupFile, restored);
+      await chmod(restored, 0o600);
+      await rm(this.hubDb, { force: true });
+      await copyFile(restored, this.hubDb);
+      await rm(restored, { force: true });
+      step('hub_db_restored', { from: this.backupFile, sha256: await sha256File(this.backupFile) });
+    }
+    const spool = await this.applySpool(this.hubDb);
+    step('spool_applied', spool);
+    const loadedBefore = journal.loadedBefore?.length ? journal.loadedBefore : this.labels;
+    for (const label of loadedBefore) await this.services.restoreLabel(label);
+    step('hub_loaded', { labels: loadedBefore });
+    const deadline = this.now() + this.facadeWaitMs;
+    while (!(await this.daemon.oldHubUp(this.port, { timeoutMs: ROLLBACK_PROBE_MS }))) {
+      if (this.now() >= deadline) {
+        const onPort = await this.#oldHubOnPort();
+        throw new HubError('HUB_ROLLBACK_HUB', onPort.hung
+          ? `the local hub is back and holds 127.0.0.1:${this.port} (pid ${onPort.pid}), but did not answer HTTP within ${this.facadeWaitMs} ms; it may still be busy, so check http://127.0.0.1:${this.port}/api/settings before restarting it`
+          : `the local hub did not come back on 127.0.0.1:${this.port}: ${onPort.detail}`);
+      }
+      await this.wait(this.pollMs);
+    }
+    step('hub_serving');
+    report.message = report.lost?.count
+      ? `Rolled back to the local hub. ${report.lost.count} write(s) made after the switch (${Object.entries(report.lost.kinds).map(([kind, n]) => `${kind} ${n}`).join(', ')}) are not in hub.db; they are listed in ${report.lost.file}.`
+      : 'Rolled back to the local hub. Nothing written after the switch was lost.';
+    await writeOwnerOnlyJson(this.journalFile, { ...journal, state: 'rolled_back', rolledBackAt: new Date(this.now()).toISOString(), rollback: report });
+    return report;
+  }
+
+  /**
+   * After the runner left the port: the seed is marked rolled back, so the next cutover replaces it with its own
+   * backup instead of serving this attempt's state (B1). Null when runner.db was never seeded.
+   */
+  async #markSeedRolledBack() {
+    const seed = this.#seed();
+    if (!seed) return null;
+    const rolledBackAt = seed.rolledBackAt || new Date(this.now()).toISOString();
+    const store = await RunnerStore.open(this.configDir);
+    try {
+      store.setMeta(SEED_META, { ...seed, rolledBackAt });
+    } finally {
+      store.close();
+    }
+    return { sha256: seed.sha256 ?? null, rolledBackAt };
+  }
+
+  /**
+   * What runner.db holds that was written after the switch: it doesn't reach hub.db, so it's listed for the user.
+   * Messages, acks, delivery receipts and registrations (M2), in time order.
+   */
+  async #exportLost(cutoverAt) {
+    const store = RunnerStore.openReadOnly(this.configDir);
+    if (!store) return { count: 0, kinds: {}, file: null };
+    const rows = [];
+    try {
+      const has = (name) => store.db.prepare("SELECT 1 AS found FROM sqlite_master WHERE type = 'table' AND name = ?").get(name);
+      const all = (kind, sql, ...args) => rows.push(...store.db.prepare(sql).all(...args).map((row) => ({ kind, ...row })));
+      if (has('messages')) all('message', 'SELECT id, sender, topic, type, body, created_at AS at FROM messages WHERE created_at >= ?', cutoverAt);
+      if (has('deliveries')) {
+        all('ack', 'SELECT message_id, recipient, acked_at AS at FROM deliveries WHERE acked_at >= ?', cutoverAt);
+        all('delivered', `SELECT message_id, recipient, delivered_via AS via, delivered_at AS at FROM deliveries
+          WHERE delivered_at >= ? AND NOT COALESCE(acked_at >= ?, 0)`, cutoverAt, cutoverAt);
+      }
+      if (has('actors') && has('pulse_registered')) {
+        all('registration', `SELECT a.id, a.role, a.node_type, a.topics, COALESCE(r.at, a.started_at) AS at
+          FROM actors a LEFT JOIN pulse_registered r ON r.id = a.id WHERE a.started_at >= ? OR r.at >= ?`, cutoverAt, cutoverAt);
+      }
+    } finally {
+      store.close();
+    }
+    rows.sort((a, b) => Number(a.at) - Number(b.at));
+    const kinds = {};
+    for (const row of rows) kinds[row.kind] = (kinds[row.kind] || 0) + 1;
+    const file = path.join(this.configDir, `rollback-lost-${this.now()}.jsonl`);
+    await writeFile(file, rows.map((row) => JSON.stringify(row)).join('\n') + (rows.length ? '\n' : ''), { mode: 0o600 });
+    return { count: rows.length, kinds, file };
+  }
+}

package/src/lib/cutoverWiring.js

@@ -0,0 +1,137 @@
+import { randomBytes } from 'node:crypto';
+import { readFile } from 'node:fs/promises';
+import path from 'node:path';
+import { fileURLToPath } from 'node:url';
+import { CloudFirstCommands } from './cloudFirstCommands.js';
+import { Cutover } from './cutover.js';
+import { HubError } from './errors.js';
+import { discoverLocalHub } from './localHub.js';
+import { ServiceManager } from './serviceManager.js';
+import { defaultSecretStore } from './secretStore.js';
+import { backupDatabase } from './sqlite.js';
+import { readOwnerOnlyJson } from './storage.js';
+import { execFileBounded, spawnBounded } from './subprocess.js';
+import { TokenStore } from './tokenStore.js';
+
+const RUNNER = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..', 'runner');
+const LSOF = '/usr/sbin/lsof';
+
+async function hubToken(hubHome) {
+  return (await readFile(path.join(hubHome, 'state', 'hub.token'), 'utf8')).trim();
+}
+
+async function local(port, route, token, { method = 'GET', body = null, timeoutMs = 3_000 } = {}) {
+  const res = await fetch(`http://127.0.0.1:${port}${route}`, {
+    method, signal: AbortSignal.timeout(timeoutMs),
+    headers: { authorization: `Bearer ${token}`, ...(body ? { 'content-type': 'application/json' } : {}) },
+    body: body ? JSON.stringify(body) : undefined,
+  });
+  return { status: res.status, body: await res.json().catch(() => null) };
+}
+
+/** Who has these files open, by `lsof -t`. Exit 1 with no output means nobody; anything else unexpected is unknown. */
+export async function lsofHolders(files, { run = execFileBounded } = {}) {
+  try {
+    const { stdout } = await run(LSOF, ['-t', ...files], { timeout: 10_000 });
+    return { pids: String(stdout).split(/\s+/).filter(Boolean).map(Number).filter(Number.isInteger) };
+  } catch (error) {
+    if (error.code === 1 && !String(error.stdout || '').trim()) return { pids: [] };
+    return { error: String(error.message || error).slice(0, 200) };
+  }
+}
+
+/**
+ * Who owns the LISTEN socket on 127.0.0.1:port, each with its parent, by `lsof -FpR` (ps is not allowed in an agent's
+ * sandbox; lsof is). Exit 1 with no output means nobody; anything else unexpected is unknown.
+ */
+export async function lsofPortOwners(port, { run = execFileBounded } = {}) {
+  try {
+    const { stdout } = await run(LSOF, ['-nP', `-iTCP@127.0.0.1:${Number(port)}`, '-sTCP:LISTEN', '-FpR'], { timeout: 10_000 });
+    const owners = [];
+    for (const line of String(stdout).split('\n')) {
+      const value = Number(line.slice(1));
+      if (!Number.isInteger(value)) continue;
+      if (line.startsWith('p') && !owners.some((o) => o.pid === value)) owners.push({ pid: value, ppid: null });
+      else if (line.startsWith('R') && owners.length) owners.at(-1).ppid ??= value;
+    }
+    return { owners };
+  } catch (error) {
+    if (error.code === 1 && !String(error.stdout || '').trim()) return { owners: [] };
+    return { error: String(error.message || error).slice(0, 200) };
+  }
+}
+
+/**
+ * After the switch: `bus ask` to the manager through the installed shim reaches the runner, a hook delivers a probe
+ * message, and (with a channel) one post lands through the facade. Nothing here talks to hub.db.
+ */
+export async function smokeTest({ hubHome, port, channel = null, run = spawnBounded, node = process.execPath }) {
+  const token = await hubToken(hubHome);
+  const probe = `cutover-probe-${randomBytes(4).toString('hex')}`;
+  const cli = (...argv) => run(node, [path.join(hubHome, 'hub.mjs'), ...argv], { timeoutMs: 15_000 }).then((r) => JSON.parse(r.stdout));
+  const checks = [];
+  const ask = await cli('bus', 'ask', '--as', probe, '--topic', 'manager', 'cutover smoke test: no answer needed');
+  if (ask.spooled || !ask.to?.includes('manager')) return { ok: false, reason: `bus ask did not reach the runner: ${JSON.stringify(ask).slice(0, 200)}` };
+  checks.push('ask');
+  await cli('bus', 'register', '--as', probe, '--role', 'CHECK');
+  // The bus refuses mail to an address it has never seen, so the probe's first hook makes it a session; the second shows the mail.
+  const hook = () => run(node, [path.join(hubHome, 'hooks', 'hub-bridge.mjs')], {
+    timeoutMs: 5_000, input: JSON.stringify({ hook_event_name: 'postToolUse', conversation_id: probe, tool_name: 'Read' }),
+  });
+  await hook();
+  const sent = await cli('bus', 'publish', '--as', 'user', '--topic', `agent/${probe}`, 'cutover hook check');
+  if (!sent.id) return { ok: false, reason: `the probe message was refused: ${JSON.stringify(sent).slice(0, 200)}` };
+  if (!String((await hook()).stdout).includes(sent.id)) return { ok: false, reason: 'the hook did not deliver the probe message' };
+  checks.push('hook');
+  if (channel) {
+    const posted = await local(port, '/api/chat/post', token, { method: 'POST', body: { ch: `task:${channel}`, text: 'Cutover smoke test: Pulse now serves this hub.' } });
+    if (posted.status !== 200) return { ok: false, reason: `the channel post failed: ${posted.status}` };
+    checks.push('channel');
+  }
+  return { ok: true, checks, probe };
+}
+
+/**
+ * Who answers on the port: the Pulse facade (it has /runner/health) or the old hub (it doesn't, and serves the API).
+ * `timeoutMs` bounds each request; rollback waits longer for a hub that is busy right after it loads.
+ */
+export function portProbes(tokenFor) {
+  return {
+    facadeUp: async (p) => {
+      try { return (await local(p, '/runner/health', await tokenFor())).status === 200; } catch { return false; }
+    },
+    oldHubUp: async (p, { timeoutMs = 3_000 } = {}) => {
+      try {
+        const token = await tokenFor();
+        return (await local(p, '/runner/health', token, { timeoutMs })).status === 404 && (await local(p, '/api/settings', token, { timeoutMs })).status === 200;
+      } catch { return false; }
+    },
+  };
+}
+
+/** The production seams of `pulse cutover`: launchd, lsof, the Phase 2 import and reconcile, and the local port. */
+export async function createCutover({ configDir, log = () => {}, port = 8890 }) {
+  const config = await readOwnerOnlyJson(path.join(configDir, 'config.json'));
+  if (!config) throw new HubError('HUB_NOT_INSTALLED', 'Pulse is not installed here; run `agentpulse pulse install`');
+  const hubHome = config.cloudFirst?.hubHome || config.mirror?.hub?.dir || (await discoverLocalHub({ config })).dir;
+  const tokenFor = () => hubToken(hubHome);
+  return new Cutover({
+    configDir, hubHome, port, log,
+    services: new ServiceManager({}),
+    probe: { holders: (files) => lsofHolders(files), portOwners: (p) => lsofPortOwners(p) },
+    cloudFirst: new CloudFirstCommands({ configDir }),
+    daemon: portProbes(tokenFor),
+    smoke: (options) => smokeTest(options),
+    backupDb: backupDatabase,
+    applySpool: (file) => spoolIntoHubDb(file, { hubHome }),
+    signIn: new TokenStore({ secretStore: defaultSecretStore(), configDir }),
+  });
+}
+
+/** Rollback's spool step: the freeze's spooled writes applied to the restored hub.db, in a child running the hub core. */
+export async function spoolIntoHubDb(file, { hubHome, env = {} }) {
+  const { stdout } = await execFileBounded(process.execPath, [path.join(RUNNER, 'spoolInto.js'), file], {
+    timeout: 120_000, env: { ...process.env, ...env, PULSE_HUB_HOME: hubHome },
+  });
+  return JSON.parse(stdout);
+}

package/src/lib/daemonLock.js

@@ -1,6 +1,7 @@
 import path from 'node:path';
 import { HubError } from './errors.js';
 import { acquireOwnerLock } from './ownerLock.js';
+import { CONNECTOR_VERSION } from './pairingLifecycle.js';
 import { ProcessIdentityProvider } from './processOwnership.js';
 import { ensureOwnerOnlyDirectory, writeOwnerOnlyJson } from './storage.js';
 
@@ -41,6 +42,7 @@ export class DaemonLock {
       pid: this.pid,
       heartbeatAt: new Date().toISOString(),
       ...status,
+      connectorVersion: CONNECTOR_VERSION,
     });
   }
 

package/src/lib/daemonLoop.js

@@ -10,7 +10,7 @@ export const MIRROR_UPDATE_STEP = 'mirror_update';
  * adds fields such as the restart state to every health write.
  */
 export function createHeartbeat({
-  readConfig, runtime, mirror, lock, ledger, ownerStore, progress = () => {}, extraHealth = () => ({}),
+  readConfig, runtime, mirror, shadow = null, runner = null, lock, ledger, ownerStore, progress = () => {}, extraHealth = () => ({}),
 }) {
   let last = null;
   const health = (config, status) => ({
@@ -56,7 +56,10 @@ export function createHeartbeat({
     }
     progress(MIRROR_UPDATE_STEP);
     // A cloud that refused the contract would refuse every batch too, so the mirror waits for the next session.
-    await mirror.update(TERMINAL_STATES.has(status.state) || status.code === 'contract_unsupported' ? null : config);
+    const usable = !(TERMINAL_STATES.has(status.state) || status.code === 'contract_unsupported');
+    await mirror.update(usable ? config : null);
+    shadow?.update(usable ? config : null);
+    runner?.update(usable ? config : null);
     progress('health');
     last = { config, status };
     await write();

package/src/lib/daemonProcess.js

@@ -17,6 +17,8 @@ import {
   Watchdog,
   watchSource,
 } from './selfRestart.js';
+import { ShadowService } from './shadowService.js';
+import { RunnerService } from '../runner/runnerService.js';
 import { readOwnerOnlyJson } from './storage.js';
 
 // The longest a healthy cycle goes between two steps: a transport retry backoff (capped at 60 s) or a held claim (35 s).
@@ -26,6 +28,8 @@ const CYCLE_INTERVAL_MS = 60_000;
 const MIRROR_INTERVAL_MS = 60_000;
 const STALL_FACTOR = 3;
 const MIRROR_LOOPS = ['mirror', 'mirror windows'];
+// The shadow feed's calls run on their own timer; they never count as progress of the job cycle.
+const SHADOW_PATHS = ['/commands', '/cloud-first', '/parity'];
 /** The failure code of a job a restart had to abort: a drain that ran out of time, or a stalled loop. */
 export const RESTART_ABORT_CODE = 'HUB_DAEMON_RESTARTING';
 const DEFAULT_TIMING = Object.freeze({
@@ -68,6 +72,8 @@ export async function runDaemon({
   signals = process,
   createDependencies = createProductionDependencies,
   createMirror = (options) => new MirrorController(options),
+  createShadow = (options) => new ShadowService(options),
+  createRunner = (options) => new RunnerService(options),
   dependencyOptions = {},
   runtimeOptions = {},
   lockOptions = {},
@@ -91,6 +97,8 @@ export async function runDaemon({
   let failure = null;
   let heartbeat = null;
   let runtime = null;
+  let shadow = null;
+  let runnerService = null;
   let stopServices = async () => {};
   let timer = null;
   let watcher = null;
@@ -212,6 +220,8 @@ export async function runDaemon({
     return {
       ...(loop ? { degraded: loop } : {}),
       ...(restarting ? { restarting: { ...restarting, draining: runtime?.activeJobIds() ?? [] } } : {}),
+      ...(shadow ? { shadow: shadow.health() } : {}),
+      ...(runnerService ? { runner: runnerService.health() } : {}),
     };
   };
 
@@ -277,7 +287,7 @@ export async function runDaemon({
       onTransportAttempt: ({ path: requestPath, phase }) => {
         const call = `${phase} ${requestPath}`;
         if (requestPath === MIRROR_PATH) mirrorProgress(call);
-        else watchdog.progress('cycle', { call });
+        else if (!SHADOW_PATHS.some((suffix) => requestPath.split('?')[0].endsWith(suffix))) watchdog.progress('cycle', { call });
       },
       ...dependencyOptions,
     });
@@ -312,10 +322,14 @@ export async function runDaemon({
         },
       },
     });
+    shadow = createShadow({ directory, transportFactory: production.transportFactory, tokenStore: production.tokenStore, log: mirrorLog });
+    runnerService = createRunner({ directory, transportFactory: production.transportFactory, log });
     heartbeat = createHeartbeat({
       readConfig: () => readOwnerOnlyJson(configFile),
       runtime,
       mirror,
+      shadow,
+      runner: runnerService,
       lock,
       ledger,
       ownerStore,
@@ -323,7 +337,7 @@ export async function runDaemon({
       extraHealth,
     });
     stopServices = async ({ abortCode }) => {
-      const results = await Promise.allSettled([mirror.stop(), runtime.shutdown({ abortCode })]);
+      const results = await Promise.allSettled([mirror.stop(), shadow.stop(), runnerService.stop(), runtime.shutdown({ abortCode })]);
       const failed = results.find((result) => result.status === 'rejected');
       if (failed) throw failed.reason;
     };

package/src/lib/daemonRuntime.js

@@ -49,6 +49,8 @@ const SETTLE_GIVE_UP_MS = 30_000;
 // 20 s Retry-After. An open session is reused, so these two set the steady call rate.
 const HEARTBEAT_INTERVAL_MS = 15_000;
 const CLAIM_SPACING_MS = 10_000;
+// With claims off a cycle is only the heartbeat, so it runs at the heartbeat's own pace.
+const CLAIMS_OFF_WAIT_MS = HEARTBEAT_INTERVAL_MS;
 // A cloud that refuses this connector's contract version is asked again this late, or at the next daemon start.
 export const CONTRACT_RETRY_MS = 60 * 60 * 1_000;
 
@@ -610,6 +612,12 @@ export class DaemonRuntime {
     await this.#flush(transport);
     if (this.stopping) return { state: 'stopping', processed: 0 };
     if (this.draining) return { state: 'draining', processed: 0 };
+    // `jobs.claim: false` keeps the session, heartbeat and mirror but takes no work, so a test connector beside the
+    // live hub can never run a hub.* job into it.
+    if (this.config.jobs?.claim === false) {
+      this.state = 'running';
+      return { state: this.state, processed: 0, claims: 'off', retryAfterMs: CLAIMS_OFF_WAIT_MS };
+    }
     this.progress('claim');
     const claimedAt = this.now();
     const claimed = await transport.poll({