@celilo/cli 0.18.0 → 0.19.0

package/src/services/events-daemon.ts

@@ -50,6 +50,8 @@ export interface InstallDaemonOptions {
   busDbPath?: string;
   /** Override the home directory used to compute install paths. */
   home?: string;
+  /** Prefix for system-scope paths. Test seam — see getDaemonUnitPath. */
+  systemRoot?: string;
 }
 
 export interface InstallDaemonResult {
@@ -61,9 +63,53 @@ export interface InstallDaemonResult {
   busDbPath: string;
   /** Explicit run-as user for system scope; undefined for user scope. */
   runAsUser?: string;
+  /**
+   * Set when the OTHER scope already has a unit installed. Both scopes use the
+   * same unit name, so installing over that produces two same-named daemons on
+   * one bus — see conflictingScopeError.
+   */
+  conflict?: { scope: SupervisorScope; unitPath: string };
   nextSteps: string[];
 }
 
+export const SUPERVISOR_SCOPES: readonly SupervisorScope[] = ['user', 'system'];
+
+/**
+ * Both scopes name the unit `celilo-events.service`, so `systemctl status
+ * celilo-events.service` and `systemctl --user status celilo-events.service`
+ * are different services that look identical in every operator-facing string.
+ *
+ * That is how celilo-mgr came to run two dispatchers for 40 days (#580, #610):
+ * Ansible installed the system unit, someone later ran `celilo events
+ * install-daemon` — which defaults to USER scope — and got a second daemon with
+ * no warning. `systemctl status celilo-events.service` then reported the dead
+ * system unit while the user-scope one served production.
+ */
+function conflictingScopeError(conflict: { scope: SupervisorScope; unitPath: string }): Error {
+  const flag = conflict.scope === 'system' ? ' --system' : '';
+  return new Error(
+    [
+      `celilo events daemon: a ${conflict.scope}-scope unit is already installed at ${conflict.unitPath}.`,
+      'Both scopes use the same unit name, so installing this one would create a SECOND dispatcher',
+      'on the same bus that `systemctl status` cannot distinguish.',
+      `Remove the other first: \`celilo events uninstall-daemon${flag}\` (and disable it in systemd),`,
+      'or keep the existing one.',
+    ].join(' '),
+  );
+}
+
+/** The other scope's unit, if it exists. */
+function findConflictingScope(
+  platform: SupervisorPlatform,
+  home: string,
+  scope: SupervisorScope,
+  systemRoot?: string,
+): { scope: SupervisorScope; unitPath: string } | undefined {
+  const other = scope === 'user' ? 'system' : 'user';
+  const found = readInstalledUnit({ platform, scope: other, home, systemRoot });
+  return found.exists ? { scope: other, unitPath: found.path } : undefined;
+}
+
 export interface UninstallDaemonResult {
   platform: SupervisorPlatform;
   scope: SupervisorScope;
@@ -90,20 +136,26 @@ export function detectPlatform(): SupervisorPlatform {
 
 /**
  * Resolve where to write the unit file for a given platform + scope.
- * `home` only matters for user scope; system paths are fixed.
+ * `home` only matters for user scope.
+ *
+ * `systemRoot` prefixes the system-scope path. It exists so a test can have
+ * BOTH scopes installed at once without writing to the real /etc — the
+ * two-units-one-name state that produced #610 is otherwise untestable, since
+ * `home` alone can only ever relocate the user unit.
  */
 export function getDaemonUnitPath(
   platform: SupervisorPlatform,
   home: string,
   scope: SupervisorScope = 'user',
+  systemRoot = '/',
 ): string {
   if (platform === 'linux') {
     return scope === 'system'
-      ? join('/etc/systemd/system', SYSTEMD_UNIT_NAME)
+      ? join(systemRoot, 'etc/systemd/system', SYSTEMD_UNIT_NAME)
       : join(home, '.config', 'systemd', 'user', SYSTEMD_UNIT_NAME);
   }
   return scope === 'system'
-    ? join('/Library/LaunchDaemons', `${LAUNCHD_LABEL}.plist`)
+    ? join(systemRoot, 'Library/LaunchDaemons', `${LAUNCHD_LABEL}.plist`)
     : join(home, 'Library', 'LaunchAgents', `${LAUNCHD_LABEL}.plist`);
 }
 
@@ -286,7 +338,7 @@ export function planDaemonInstall(opts: InstallDaemonOptions = {}): InstallDaemo
   const concurrency = opts.concurrency ?? 4;
   const runAsUser = scope === 'system' ? resolveRunAsUser(busDbPath, opts.runAsUser) : undefined;
 
-  const unitPath = getDaemonUnitPath(platform, home, scope);
+  const unitPath = getDaemonUnitPath(platform, home, scope, opts.systemRoot);
   const unitInputs: UnitInputs = {
     celiloPath,
     busDbPath,
@@ -307,6 +359,7 @@ export function planDaemonInstall(opts: InstallDaemonOptions = {}): InstallDaemo
     celiloPath,
     busDbPath,
     runAsUser,
+    conflict: findConflictingScope(platform, home, scope, opts.systemRoot),
     nextSteps: nextStepsFor(platform, scope, unitPath),
   };
 }
@@ -320,18 +373,27 @@ export function planDaemonInstall(opts: InstallDaemonOptions = {}): InstallDaemo
  */
 export function installDaemon(opts: InstallDaemonOptions = {}): InstallDaemonResult {
   const plan = planDaemonInstall(opts);
+  // Refuse on the WRITE path only. `--print` renders without creating anything,
+  // and the celilo-mgmt Ansible role captures it — throwing there would wedge
+  // the deploy of the very tool an operator needs to clean the conflict up.
+  if (plan.conflict) throw conflictingScopeError(plan.conflict);
   mkdirSync(dirname(plan.unitPath), { recursive: true });
   writeFileSync(plan.unitPath, plan.unitContent, { mode: 0o644 });
   return plan;
 }
 
 export function uninstallDaemon(
-  opts: { platform?: SupervisorPlatform; scope?: SupervisorScope; home?: string } = {},
+  opts: {
+    platform?: SupervisorPlatform;
+    scope?: SupervisorScope;
+    home?: string;
+    systemRoot?: string;
+  } = {},
 ): UninstallDaemonResult {
   const platform = opts.platform ?? detectPlatform();
   const scope = opts.scope ?? 'user';
   const home = opts.home ?? homedir();
-  const unitPath = getDaemonUnitPath(platform, home, scope);
+  const unitPath = getDaemonUnitPath(platform, home, scope, opts.systemRoot);
 
   let removed = false;
   if (existsSync(unitPath)) {
@@ -366,13 +428,238 @@ export function uninstallDaemon(
   return { platform, scope, unitPath, removed, nextSteps };
 }
 
+// --- restart (celilo#604) ---------------------------------------------
+//
+// `install-daemon` deliberately never touches supervisor state, so the only
+// supported restart was SSH + systemctl — which the "operate celilo through
+// celilo" model forbids. Restart is the one supervisor verb that has to be a
+// celilo command, because the upgrade path needs it.
+//
+// The trap: on celilo-mgr the running dispatcher was an ORPHAN (PPID 1, not the
+// unit's child). `systemctl restart` could not kill it, the unit crash-looped
+// against the one-dispatcher-per-bus guard (#584), and systemctl reported
+// `activating` while the OLD code kept serving. So a restart that shells out and
+// returns is worthless: it must reconcile the orphan and then verify the bus.
+
+export interface RestartDaemonResult {
+  platform: SupervisorPlatform;
+  scope: SupervisorScope;
+  unitPath: string;
+  /** Live dispatchers stopped because the supervisor did not own them. */
+  orphansKilled: number[];
+  /** Version the restarted dispatcher must report (installed @celilo/event-bus). */
+  expectedVersion: string;
+  /** The dispatcher now serving the bus. */
+  dispatcher: { pid: number; version: string | null };
+}
+
+export interface RestartDaemonDeps {
+  /** Live dispatchers on the bus, by pid + the code version each reports. */
+  liveDispatchers: () => Array<{ pid: number; version: string | null }>;
+  /** The pid the supervisor owns for the unit, or null if it owns none. */
+  supervisorPid: () => number | null;
+  /** Stop a process the supervisor does not own. */
+  kill: (pid: number, signal: NodeJS.Signals) => void;
+  /** Ask the supervisor to (re)start the unit. Throws on failure. */
+  restartUnit: () => void;
+  sleep: (ms: number) => Promise<void>;
+}
+
+export interface RestartDaemonOptions {
+  platform?: SupervisorPlatform;
+  scope?: SupervisorScope;
+  home?: string;
+  /** Version the restarted dispatcher must report before we call it a success. */
+  expectedVersion: string;
+  /** How long to wait for the new dispatcher to appear on the bus. */
+  timeoutMs?: number;
+  /** Poll interval while waiting. */
+  pollMs?: number;
+}
+
+/**
+ * Which live dispatchers block the unit from starting: every one whose pid the
+ * supervisor does not own. Pure — the orphan trap is decided here so it can be
+ * tested without a systemd.
+ */
+export function orphanDispatcherPids(
+  live: Array<{ pid: number }>,
+  supervisorPid: number | null,
+): number[] {
+  return live.filter((d) => d.pid !== supervisorPid).map((d) => d.pid);
+}
+
+/**
+ * Pick the scope to act on: whichever scope has a unit installed. Explicit wins;
+ * with units in both scopes, system wins (the management-plane shape).
+ */
+/** Is a supervisor unit present in either scope? */
+export function unitInstalledInAnyScope(platform?: SupervisorPlatform, home?: string): boolean {
+  return (['system', 'user'] as const).some(
+    (scope) => readInstalledUnit({ scope, home, platform }).exists,
+  );
+}
+
+export function resolveRestartScope(
+  opts: { platform?: SupervisorPlatform; home?: string; scope?: SupervisorScope } = {},
+): SupervisorScope {
+  if (opts.scope) return opts.scope;
+  const probe = (scope: SupervisorScope) =>
+    readInstalledUnit({ scope, home: opts.home, platform: opts.platform }).exists;
+  if (probe('system')) return 'system';
+  if (probe('user')) return 'user';
+  throw new Error(
+    'celilo events restart-daemon: no supervisor unit installed (user or system scope). Run `celilo events install-daemon` first.',
+  );
+}
+
+/**
+ * Stop any unmanaged dispatcher, restart the unit, and confirm on the BUS — not
+ * from the supervisor's exit code — that a new process is live on the expected
+ * version. Anything less reports success into the crash loop described above.
+ */
+export async function restartDaemon(
+  opts: RestartDaemonOptions,
+  deps: RestartDaemonDeps,
+): Promise<RestartDaemonResult> {
+  const platform = opts.platform ?? detectPlatform();
+  const scope = resolveRestartScope({ platform, home: opts.home, scope: opts.scope });
+  const unitPath = getDaemonUnitPath(platform, opts.home ?? homedir(), scope);
+  const timeoutMs = opts.timeoutMs ?? 60_000;
+  const pollMs = opts.pollMs ?? 1000;
+
+  const before = deps.liveDispatchers();
+  const orphans = orphanDispatcherPids(before, deps.supervisorPid());
+  for (const pid of orphans) {
+    deps.kill(pid, 'SIGTERM');
+  }
+  // Wait for the guard's view to clear before restarting, else the unit starts
+  // into `another dispatcher is already running` and auto-restart backoff eats
+  // the window. SIGKILL whatever refuses to go.
+  const graceUntil = Date.now() + 10_000;
+  while (orphans.length > 0 && Date.now() < graceUntil) {
+    if (deps.liveDispatchers().every((d) => !orphans.includes(d.pid))) break;
+    await deps.sleep(pollMs);
+  }
+  for (const pid of deps.liveDispatchers().map((d) => d.pid)) {
+    if (orphans.includes(pid)) deps.kill(pid, 'SIGKILL');
+  }
+
+  deps.restartUnit();
+
+  const beforePids = new Set(before.map((d) => d.pid));
+  const deadline = Date.now() + timeoutMs;
+  let last: Array<{ pid: number; version: string | null }> = [];
+  while (Date.now() < deadline) {
+    last = deps.liveDispatchers();
+    const fresh = last.filter((d) => !beforePids.has(d.pid));
+    if (last.length === 1 && fresh.length === 1 && fresh[0]?.version === opts.expectedVersion) {
+      return {
+        platform,
+        scope,
+        unitPath,
+        orphansKilled: orphans,
+        expectedVersion: opts.expectedVersion,
+        dispatcher: fresh[0],
+      };
+    }
+    await deps.sleep(pollMs);
+  }
+
+  const seen =
+    last.length === 0
+      ? 'no dispatcher is live on the bus'
+      : `live: ${last.map((d) => `pid ${d.pid} (v${d.version ?? '?'})`).join(', ')}`;
+  throw new Error(
+    `celilo events restart-daemon: the supervisor was asked to restart ${unitPath}, but no dispatcher on v${opts.expectedVersion} came up within ${Math.round(timeoutMs / 1000)}s — ${seen}. The old code may still be serving. Check \`celilo events status\` and the unit's logs.`,
+  );
+}
+
+/**
+ * systemctl/launchctl argv for this platform+scope — `restart` and MainPID.
+ *
+ * System scope goes through `sudo`, and that is not optional on the box this
+ * exists for: the unit is root-owned and the command runs as the unprivileged
+ * `celilo` user (api-serve's child, the wrapper's sudo-drop target). Without
+ * sudo, every apt-upgrade on celilo-mgr would take the honest-but-useless
+ * branch — "packages upgraded, dispatcher still on old code" — forever.
+ * `celilo-bootstrap` ships the scoped grant (/etc/sudoers.d/celilo-events-restart)
+ * for exactly these two invocations. Same shape as `install-daemon`'s own
+ * printed next-steps, which already say `sudo systemctl` for system scope.
+ */
+export function supervisorCommands(
+  platform: SupervisorPlatform,
+  scope: SupervisorScope,
+): { restart: string[]; mainPid: string[] | null } {
+  if (platform === 'linux') {
+    const sc = scope === 'system' ? ['sudo', 'systemctl'] : ['systemctl', '--user'];
+    return {
+      restart: [...sc, 'restart', SYSTEMD_UNIT_NAME],
+      mainPid: [...sc, 'show', SYSTEMD_UNIT_NAME, '-p', 'MainPID', '--value'],
+    };
+  }
+  const target =
+    scope === 'system'
+      ? `system/${LAUNCHD_LABEL}`
+      : `gui/${process.getuid?.() ?? 501}/${LAUNCHD_LABEL}`;
+  return {
+    // kickstart -k restarts a loaded job; bootstrap first if it was never loaded.
+    restart:
+      scope === 'system'
+        ? ['sudo', 'launchctl', 'kickstart', '-k', target]
+        : ['launchctl', 'kickstart', '-k', target],
+    // launchctl print's pid line is not machine-stable across releases; treat
+    // launchd as "owns nothing we can name" and let the bus decide.
+    // ponytail: no MainPID probe on darwin — celilo-mgr is Linux; add if a mac
+    // ever runs the management plane.
+    mainPid: null,
+  };
+}
+
+/**
+ * The pid the supervisor currently owns for this scope's unit, or null when
+ * that can't be established (launchd, systemd absent, unit inactive, user bus
+ * unreachable). `MainPID=0` means inactive — reported as null, not 0.
+ *
+ * Null is "don't know", never "not supervised": a caller must not accuse a
+ * dispatcher of being an orphan on the strength of a failed probe.
+ *
+ * The argv comes from supervisorCommands (celilo#604) rather than being rebuilt
+ * here — that is where the sudo-for-system-scope decision lives, and two copies
+ * of it would drift.
+ */
+export function unitMainPid(
+  scope: SupervisorScope,
+  platform: SupervisorPlatform = detectPlatform(),
+): number | null {
+  const { mainPid } = supervisorCommands(platform, scope);
+  if (!mainPid) return null;
+  const [cmd, ...args] = mainPid;
+  if (!cmd) return null;
+  try {
+    const out = execFileSync(cmd, args, {
+      encoding: 'utf-8',
+      stdio: ['ignore', 'pipe', 'ignore'],
+    });
+    const pid = Number(out.trim());
+    return Number.isFinite(pid) && pid > 0 ? pid : null;
+  } catch {
+    return null;
+  }
+}
+
 export function readInstalledUnit(
-  opts: { platform?: SupervisorPlatform; scope?: SupervisorScope; home?: string } = {},
+  opts: {
+    platform?: SupervisorPlatform;
+    scope?: SupervisorScope;
+    home?: string;
+    systemRoot?: string;
+  } = {},
 ): { exists: true; path: string; content: string } | { exists: false; path: string } {
   const platform = opts.platform ?? detectPlatform();
   const scope = opts.scope ?? 'user';
   const home = opts.home ?? homedir();
-  const unitPath = getDaemonUnitPath(platform, home, scope);
+  const unitPath = getDaemonUnitPath(platform, home, scope, opts.systemRoot);
   if (!existsSync(unitPath)) return { exists: false, path: unitPath };
   return { exists: true, path: unitPath, content: readFileSync(unitPath, 'utf-8') };
 }

package/src/services/fleet-checks.test.ts

@@ -39,9 +39,9 @@ function seedHeartbeat(
   );
 }
 
-/** Write a supervisor unit file so readInstalledUnit('user') sees it. */
-function installFakeUnit(home: string): void {
-  const path = getDaemonUnitPath('linux', home, 'user');
+/** Write a supervisor unit file so readInstalledUnit(scope) sees it. */
+function installFakeUnit(home: string, scope: 'user' | 'system' = 'user', systemRoot = '/'): void {
+  const path = getDaemonUnitPath('linux', home, scope, systemRoot);
   mkdirSync(dirname(path), { recursive: true });
   writeFileSync(path, '[Unit]\nDescription=fake\n');
 }
@@ -114,6 +114,51 @@ describe('checkDispatcher', () => {
     expect(f.detail.join(' ')).toContain('not under a supervisor');
   });
 
+  // #610 — celilo-mgr had a dead system unit and a live user-scope unit of the
+  // SAME name. The old file-exists test called that "supervised".
+  it('fails when the running dispatcher is not the pid any installed unit supervises', () => {
+    seedHeartbeat(bus, { startedAt: now - MINUTE, lastHeartbeat: now - 1000, pid: 588704 });
+    installFakeUnit(home);
+    const f = checkDispatcher(bus, {
+      now: now,
+      home,
+      platform: 'linux',
+      unitMainPid: () => 3639051, // systemd supervises a different process
+    });
+    expect(f.status).toBe('fail');
+    expect(f.detail.join(' ')).toContain('not the process any installed unit supervises');
+  });
+
+  it('fails when both a user-scope and a system-scope unit are installed', () => {
+    seedHeartbeat(bus, { startedAt: now - MINUTE, lastHeartbeat: now - 1000, pid: 4242 });
+    installFakeUnit(home);
+    installFakeUnit(home, 'system', dir);
+    const f = checkDispatcher(bus, {
+      now: now,
+      home,
+      systemRoot: dir,
+      platform: 'linux',
+      unitMainPid: () => 4242,
+    });
+    expect(f.status).toBe('fail');
+    expect(f.detail.join(' ')).toContain('same unit name, different services');
+  });
+
+  // Null is ignorance, not evidence: a systemd probe that fails must not
+  // manufacture an orphan report.
+  it('makes no supervision claim when the unit pid cannot be determined', () => {
+    seedHeartbeat(bus, { startedAt: now - MINUTE, lastHeartbeat: now - 1000, pid: 4242 });
+    installFakeUnit(home);
+    const f = checkDispatcher(bus, {
+      now: now,
+      home,
+      platform: 'linux',
+      installedCodeMtimeMs: now - 2 * MINUTE,
+      unitMainPid: () => null,
+    });
+    expect(f.status).toBe('ok');
+  });
+
   it('warns when the dispatcher started before the installed code (stale)', () => {
     seedHeartbeat(bus, { startedAt: now - 10 * MINUTE, lastHeartbeat: now - 1000 });
     installFakeUnit(home);
@@ -425,7 +470,7 @@ describe('checkSubscribers + checkCapabilityProviders', () => {
     it('is ok when every schema table is present (fresh migrated DB)', () => {
       const f = checkSchemaDrift(db);
       expect(f.status).toBe('ok');
-      expect(f.summary).toContain('schema tables present');
+      expect(f.summary).toContain('schema tables');
     });
 
     it('fails and names a table the running CLI expects but the DB lacks', () => {
@@ -435,6 +480,32 @@ describe('checkSubscribers + checkCapabilityProviders', () => {
       expect(f.detail.join(' ')).toContain('dns_internal_records');
       expect(f.remediation).toContain('migrations');
     });
+
+    // celilo#604: this is the state the rollout could not check. Every table
+    // is present, one MIGRATED COLUMN is not, and the doctor must not call
+    // that "migrations applied".
+    it('fails and names a migrated COLUMN the DB lacks, with every table present', () => {
+      db.$client.run('ALTER TABLE backups DROP COLUMN pid');
+      const f = checkSchemaDrift(db);
+      expect(f.status).toBe('fail');
+      expect(f.detail.join(' ')).toContain('backups.pid');
+      expect(f.summary).not.toContain('present');
+    });
+
+    it('says it checked columns, not only tables', () => {
+      const f = checkSchemaDrift(db);
+      expect(f.status).toBe('ok');
+      expect(f.summary).toContain('columns present');
+    });
+
+    it('fails when a journal migration has not been applied on this box', () => {
+      db.$client.run(
+        'DELETE FROM `__drizzle_migrations` WHERE created_at = (SELECT MAX(created_at) FROM `__drizzle_migrations`)',
+      );
+      const f = checkSchemaDrift(db);
+      expect(f.status).toBe('fail');
+      expect(f.detail.join(' ')).toContain('unapplied migration');
+    });
   });
 });
 

package/src/services/fleet-checks.ts

@@ -20,7 +20,8 @@
 import type { Bus } from '@celilo/event-bus';
 import { inArray } from 'drizzle-orm';
 import { getModuleStoragePath } from '../config/paths';
-import type { DbClient } from '../db/client';
+import { type DbClient, findMigrationsFolder } from '../db/client';
+import { getMigrationStatus } from '../db/migration-status';
 import { capabilities as capabilitiesTable, modules } from '../db/schema';
 import { findSchemaDrift } from '../db/schema-introspection';
 import { loadControlPlaneSubnet, resolveFirewallNatIp } from '../hooks/capability-loader';
@@ -30,7 +31,13 @@ import type { ModuleManifest } from '../manifest/schema';
 const CONTROL_PLANE_MODULE = 'celilo-mgmt';
 import { getModuleSystems } from './deployed-systems';
 import { listDnsInternalRecords } from './dns-internal-records';
-import { type SupervisorPlatform, readInstalledUnit } from './events-daemon';
+import {
+  SUPERVISOR_SCOPES,
+  type SupervisorPlatform,
+  type SupervisorScope,
+  readInstalledUnit,
+  unitMainPid,
+} from './events-daemon';
 import { resolveSubscription } from './module-subscriptions';
 
 /**
@@ -107,26 +114,43 @@ function worst(statuses: FleetFindingStatus[]): FleetFindingStatus {
  * its schema is current, so presence is the honest signal.
  */
 export function checkSchemaDrift(db: DbClient): FleetFinding {
-  const { missingTables, missingColumns, tableCount } = findSchemaDrift(db.$client);
+  const { missingTables, missingColumns, tableCount, columnCount } = findSchemaDrift(db.$client);
 
   const detail: string[] = [];
   if (missingTables.length > 0) detail.push(`missing table(s): ${missingTables.join(', ')}`);
   if (missingColumns.length > 0) detail.push(`missing column(s): ${missingColumns.join(', ')}`);
+
+  // Also name unapplied migrations. Presence is the honest signal for schema
+  // objects, but a migration can carry an index or a data fix that presence
+  // can't see — and an operator reading "migrations applied" deserves to know
+  // when some aren't (celilo#604). Best-effort: an install layout where the
+  // journal can't be found must not fail the check.
+  let pending: string[] = [];
+  try {
+    pending = getMigrationStatus(db.$client, findMigrationsFolder()).pending;
+  } catch {
+    // No journal reachable — the presence check above still stands.
+  }
+  if (pending.length > 0) detail.push(`unapplied migration(s): ${pending.join(', ')}`);
+
   const status: FleetFindingStatus = detail.length > 0 ? 'fail' : 'ok';
 
   return {
     id: 'schema',
     title: 'database schema matches the running CLI (migrations applied)',
     status,
+    // Say tables AND columns: "all 35 tables present" reads as though columns
+    // went unchecked, which is what sent a rollout to sqlite3 over SSH to
+    // confirm a column migration the doctor had in fact already verified.
     summary:
       status === 'ok'
-        ? `all ${tableCount} schema tables present`
+        ? `all ${tableCount} schema tables and ${columnCount} columns present, no unapplied migrations`
         : 'database schema is behind the running CLI — migrations not applied',
     detail,
     remediation:
       status === 'ok'
         ? null
-        : 'run `celilo system migrate` to apply pending migrations on this box — see ISS-0100',
+        : 'run `celilo system migrate` to apply pending migrations on this box (`celilo system migrate --status` names them) — see ISS-0100',
     autoFixable: false,
   };
 }
@@ -152,6 +176,14 @@ export interface DispatcherCheckOptions {
   /** Override for readInstalledUnit — tests point this at a temp home. */
   home?: string;
   platform?: SupervisorPlatform;
+  /** Prefix for system-scope unit paths. Test seam — see getDaemonUnitPath. */
+  systemRoot?: string;
+  /**
+   * Which pid each scope's unit supervises. Injected so the check is testable
+   * without systemd. Returning null means "can't tell" — the check then makes
+   * no supervision claim rather than guessing.
+   */
+  unitMainPid?: (scope: SupervisorScope) => number | null;
 }
 
 /**
@@ -223,16 +255,54 @@ export function checkDispatcher(bus: Bus, opts: DispatcherCheckOptions = {}): Fl
     );
   }
 
-  // (2) supervised — a unit file exists (user or system scope). A
-  // running dispatcher with NO unit is the orphan case: works now, gone
-  // after reboot.
-  const supervised =
-    readInstalledUnit({ scope: 'user', home: opts.home, platform: opts.platform }).exists ||
-    readInstalledUnit({ scope: 'system', home: opts.home, platform: opts.platform }).exists;
-  if (!supervised) {
+  // (2) supervised — not just "a unit file exists on disk", but "the process
+  // that is actually running IS the one an installed unit supervises".
+  //
+  // The file-exists test this replaces reported green on celilo-mgr while the
+  // system unit was dead and a user-scope unit of the SAME NAME served
+  // production (#610). A check whose entire job is catching an unsupervised
+  // dispatcher cannot be satisfied by a file nobody is running.
+  const installedScopes = SUPERVISOR_SCOPES.filter(
+    (scope) =>
+      readInstalledUnit({
+        scope,
+        home: opts.home,
+        platform: opts.platform,
+        systemRoot: opts.systemRoot,
+      }).exists,
+  );
+  if (installedScopes.length === 0) {
     statuses.push('warn');
     detail.push('not under a supervisor unit — will not survive a reboot (orphan process)');
     remediations.push('`celilo events install-daemon` then enable the unit so it is supervised');
+  } else {
+    if (installedScopes.length > 1) {
+      statuses.push('fail');
+      detail.push(
+        'both a user-scope AND a system-scope unit are installed — same unit name, different services; ' +
+          'one will lose the race on every boot and retry forever',
+      );
+      remediations.push(
+        'keep exactly one: `celilo events uninstall-daemon` (user) or `celilo events uninstall-daemon --system`, and disable it in systemd',
+      );
+    }
+    // Only accuse when systemd actually answered. A null probe is ignorance,
+    // not evidence of an orphan.
+    const probe =
+      opts.unitMainPid ?? ((scope: SupervisorScope) => unitMainPid(scope, opts.platform));
+    const supervisedPids = installedScopes
+      .map((scope) => ({ scope, pid: probe(scope) }))
+      .filter((entry): entry is { scope: SupervisorScope; pid: number } => entry.pid !== null);
+    if (supervisedPids.length > 0 && !supervisedPids.some((entry) => entry.pid === hb.pid)) {
+      statuses.push('fail');
+      detail.push(
+        `the running dispatcher (pid ${hb.pid}) is not the process any installed unit supervises ` +
+          `(${supervisedPids.map((e) => `${e.scope}=${e.pid}`).join(', ')}) — restarting the unit will not restart it`,
+      );
+      remediations.push(
+        'stop the unsupervised process and let the unit own the dispatcher, or reinstall the unit for the scope that is actually running it',
+      );
+    }
   }
 
   // (3) current — started before the installed code was last written ⇒

package/src/services/interview-errors.ts

@@ -0,0 +1,20 @@
+/**
+ * The one error type that means "this interview question could not be
+ * answered". Its own module so `responder-probe` (no responder listening) and
+ * `bus-interview` (a responder replied that it couldn't decide) can both throw
+ * it without importing each other.
+ *
+ * Callers catch this to distinguish an *unanswered* question from an answered
+ * one — the distinction `module update` conflated when it reported a breaking
+ * update as "operator declined" that no operator had ever seen.
+ */
+export class InterviewUnansweredError extends Error {
+  constructor(
+    /** The interview event type that went unanswered, e.g. `interview.required.<scope>.<key>`. */
+    readonly queryType: string,
+    message: string,
+  ) {
+    super(message);
+    this.name = 'InterviewUnansweredError';
+  }
+}