@celilo/cli 0.18.0 → 0.20.0

package/src/services/fleet-checks.test.ts

@@ -39,9 +39,9 @@ function seedHeartbeat(
   );
 }
 
-/** Write a supervisor unit file so readInstalledUnit('user') sees it. */
-function installFakeUnit(home: string): void {
-  const path = getDaemonUnitPath('linux', home, 'user');
+/** Write a supervisor unit file so readInstalledUnit(scope) sees it. */
+function installFakeUnit(home: string, scope: 'user' | 'system' = 'user', systemRoot = '/'): void {
+  const path = getDaemonUnitPath('linux', home, scope, systemRoot);
   mkdirSync(dirname(path), { recursive: true });
   writeFileSync(path, '[Unit]\nDescription=fake\n');
 }
@@ -114,6 +114,51 @@ describe('checkDispatcher', () => {
     expect(f.detail.join(' ')).toContain('not under a supervisor');
   });
 
+  // #610 — celilo-mgr had a dead system unit and a live user-scope unit of the
+  // SAME name. The old file-exists test called that "supervised".
+  it('fails when the running dispatcher is not the pid any installed unit supervises', () => {
+    seedHeartbeat(bus, { startedAt: now - MINUTE, lastHeartbeat: now - 1000, pid: 588704 });
+    installFakeUnit(home);
+    const f = checkDispatcher(bus, {
+      now: now,
+      home,
+      platform: 'linux',
+      unitMainPid: () => 3639051, // systemd supervises a different process
+    });
+    expect(f.status).toBe('fail');
+    expect(f.detail.join(' ')).toContain('not the process any installed unit supervises');
+  });
+
+  it('fails when both a user-scope and a system-scope unit are installed', () => {
+    seedHeartbeat(bus, { startedAt: now - MINUTE, lastHeartbeat: now - 1000, pid: 4242 });
+    installFakeUnit(home);
+    installFakeUnit(home, 'system', dir);
+    const f = checkDispatcher(bus, {
+      now: now,
+      home,
+      systemRoot: dir,
+      platform: 'linux',
+      unitMainPid: () => 4242,
+    });
+    expect(f.status).toBe('fail');
+    expect(f.detail.join(' ')).toContain('same unit name, different services');
+  });
+
+  // Null is ignorance, not evidence: a systemd probe that fails must not
+  // manufacture an orphan report.
+  it('makes no supervision claim when the unit pid cannot be determined', () => {
+    seedHeartbeat(bus, { startedAt: now - MINUTE, lastHeartbeat: now - 1000, pid: 4242 });
+    installFakeUnit(home);
+    const f = checkDispatcher(bus, {
+      now: now,
+      home,
+      platform: 'linux',
+      installedCodeMtimeMs: now - 2 * MINUTE,
+      unitMainPid: () => null,
+    });
+    expect(f.status).toBe('ok');
+  });
+
   it('warns when the dispatcher started before the installed code (stale)', () => {
     seedHeartbeat(bus, { startedAt: now - 10 * MINUTE, lastHeartbeat: now - 1000 });
     installFakeUnit(home);
@@ -425,7 +470,7 @@ describe('checkSubscribers + checkCapabilityProviders', () => {
     it('is ok when every schema table is present (fresh migrated DB)', () => {
       const f = checkSchemaDrift(db);
       expect(f.status).toBe('ok');
-      expect(f.summary).toContain('schema tables present');
+      expect(f.summary).toContain('schema tables');
     });
 
     it('fails and names a table the running CLI expects but the DB lacks', () => {
@@ -435,6 +480,32 @@ describe('checkSubscribers + checkCapabilityProviders', () => {
       expect(f.detail.join(' ')).toContain('dns_internal_records');
       expect(f.remediation).toContain('migrations');
     });
+
+    // celilo#604: this is the state the rollout could not check. Every table
+    // is present, one MIGRATED COLUMN is not, and the doctor must not call
+    // that "migrations applied".
+    it('fails and names a migrated COLUMN the DB lacks, with every table present', () => {
+      db.$client.run('ALTER TABLE backups DROP COLUMN pid');
+      const f = checkSchemaDrift(db);
+      expect(f.status).toBe('fail');
+      expect(f.detail.join(' ')).toContain('backups.pid');
+      expect(f.summary).not.toContain('present');
+    });
+
+    it('says it checked columns, not only tables', () => {
+      const f = checkSchemaDrift(db);
+      expect(f.status).toBe('ok');
+      expect(f.summary).toContain('columns present');
+    });
+
+    it('fails when a journal migration has not been applied on this box', () => {
+      db.$client.run(
+        'DELETE FROM `__drizzle_migrations` WHERE created_at = (SELECT MAX(created_at) FROM `__drizzle_migrations`)',
+      );
+      const f = checkSchemaDrift(db);
+      expect(f.status).toBe('fail');
+      expect(f.detail.join(' ')).toContain('unapplied migration');
+    });
   });
 });
 

package/src/services/fleet-checks.ts

@@ -20,7 +20,8 @@
 import type { Bus } from '@celilo/event-bus';
 import { inArray } from 'drizzle-orm';
 import { getModuleStoragePath } from '../config/paths';
-import type { DbClient } from '../db/client';
+import { type DbClient, findMigrationsFolder } from '../db/client';
+import { getMigrationStatus } from '../db/migration-status';
 import { capabilities as capabilitiesTable, modules } from '../db/schema';
 import { findSchemaDrift } from '../db/schema-introspection';
 import { loadControlPlaneSubnet, resolveFirewallNatIp } from '../hooks/capability-loader';
@@ -30,7 +31,13 @@ import type { ModuleManifest } from '../manifest/schema';
 const CONTROL_PLANE_MODULE = 'celilo-mgmt';
 import { getModuleSystems } from './deployed-systems';
 import { listDnsInternalRecords } from './dns-internal-records';
-import { type SupervisorPlatform, readInstalledUnit } from './events-daemon';
+import {
+  SUPERVISOR_SCOPES,
+  type SupervisorPlatform,
+  type SupervisorScope,
+  readInstalledUnit,
+  unitMainPid,
+} from './events-daemon';
 import { resolveSubscription } from './module-subscriptions';
 
 /**
@@ -107,26 +114,43 @@ function worst(statuses: FleetFindingStatus[]): FleetFindingStatus {
  * its schema is current, so presence is the honest signal.
  */
 export function checkSchemaDrift(db: DbClient): FleetFinding {
-  const { missingTables, missingColumns, tableCount } = findSchemaDrift(db.$client);
+  const { missingTables, missingColumns, tableCount, columnCount } = findSchemaDrift(db.$client);
 
   const detail: string[] = [];
   if (missingTables.length > 0) detail.push(`missing table(s): ${missingTables.join(', ')}`);
   if (missingColumns.length > 0) detail.push(`missing column(s): ${missingColumns.join(', ')}`);
+
+  // Also name unapplied migrations. Presence is the honest signal for schema
+  // objects, but a migration can carry an index or a data fix that presence
+  // can't see — and an operator reading "migrations applied" deserves to know
+  // when some aren't (celilo#604). Best-effort: an install layout where the
+  // journal can't be found must not fail the check.
+  let pending: string[] = [];
+  try {
+    pending = getMigrationStatus(db.$client, findMigrationsFolder()).pending;
+  } catch {
+    // No journal reachable — the presence check above still stands.
+  }
+  if (pending.length > 0) detail.push(`unapplied migration(s): ${pending.join(', ')}`);
+
   const status: FleetFindingStatus = detail.length > 0 ? 'fail' : 'ok';
 
   return {
     id: 'schema',
     title: 'database schema matches the running CLI (migrations applied)',
     status,
+    // Say tables AND columns: "all 35 tables present" reads as though columns
+    // went unchecked, which is what sent a rollout to sqlite3 over SSH to
+    // confirm a column migration the doctor had in fact already verified.
     summary:
       status === 'ok'
-        ? `all ${tableCount} schema tables present`
+        ? `all ${tableCount} schema tables and ${columnCount} columns present, no unapplied migrations`
         : 'database schema is behind the running CLI — migrations not applied',
     detail,
     remediation:
       status === 'ok'
         ? null
-        : 'run `celilo system migrate` to apply pending migrations on this box — see ISS-0100',
+        : 'run `celilo system migrate` to apply pending migrations on this box (`celilo system migrate --status` names them) — see ISS-0100',
     autoFixable: false,
   };
 }
@@ -152,6 +176,14 @@ export interface DispatcherCheckOptions {
   /** Override for readInstalledUnit — tests point this at a temp home. */
   home?: string;
   platform?: SupervisorPlatform;
+  /** Prefix for system-scope unit paths. Test seam — see getDaemonUnitPath. */
+  systemRoot?: string;
+  /**
+   * Which pid each scope's unit supervises. Injected so the check is testable
+   * without systemd. Returning null means "can't tell" — the check then makes
+   * no supervision claim rather than guessing.
+   */
+  unitMainPid?: (scope: SupervisorScope) => number | null;
 }
 
 /**
@@ -223,16 +255,54 @@ export function checkDispatcher(bus: Bus, opts: DispatcherCheckOptions = {}): Fl
     );
   }
 
-  // (2) supervised — a unit file exists (user or system scope). A
-  // running dispatcher with NO unit is the orphan case: works now, gone
-  // after reboot.
-  const supervised =
-    readInstalledUnit({ scope: 'user', home: opts.home, platform: opts.platform }).exists ||
-    readInstalledUnit({ scope: 'system', home: opts.home, platform: opts.platform }).exists;
-  if (!supervised) {
+  // (2) supervised — not just "a unit file exists on disk", but "the process
+  // that is actually running IS the one an installed unit supervises".
+  //
+  // The file-exists test this replaces reported green on celilo-mgr while the
+  // system unit was dead and a user-scope unit of the SAME NAME served
+  // production (#610). A check whose entire job is catching an unsupervised
+  // dispatcher cannot be satisfied by a file nobody is running.
+  const installedScopes = SUPERVISOR_SCOPES.filter(
+    (scope) =>
+      readInstalledUnit({
+        scope,
+        home: opts.home,
+        platform: opts.platform,
+        systemRoot: opts.systemRoot,
+      }).exists,
+  );
+  if (installedScopes.length === 0) {
     statuses.push('warn');
     detail.push('not under a supervisor unit — will not survive a reboot (orphan process)');
     remediations.push('`celilo events install-daemon` then enable the unit so it is supervised');
+  } else {
+    if (installedScopes.length > 1) {
+      statuses.push('fail');
+      detail.push(
+        'both a user-scope AND a system-scope unit are installed — same unit name, different services; ' +
+          'one will lose the race on every boot and retry forever',
+      );
+      remediations.push(
+        'keep exactly one: `celilo events uninstall-daemon` (user) or `celilo events uninstall-daemon --system`, and disable it in systemd',
+      );
+    }
+    // Only accuse when systemd actually answered. A null probe is ignorance,
+    // not evidence of an orphan.
+    const probe =
+      opts.unitMainPid ?? ((scope: SupervisorScope) => unitMainPid(scope, opts.platform));
+    const supervisedPids = installedScopes
+      .map((scope) => ({ scope, pid: probe(scope) }))
+      .filter((entry): entry is { scope: SupervisorScope; pid: number } => entry.pid !== null);
+    if (supervisedPids.length > 0 && !supervisedPids.some((entry) => entry.pid === hb.pid)) {
+      statuses.push('fail');
+      detail.push(
+        `the running dispatcher (pid ${hb.pid}) is not the process any installed unit supervises ` +
+          `(${supervisedPids.map((e) => `${e.scope}=${e.pid}`).join(', ')}) — restarting the unit will not restart it`,
+      );
+      remediations.push(
+        'stop the unsupervised process and let the unit own the dispatcher, or reinstall the unit for the scope that is actually running it',
+      );
+    }
   }
 
   // (3) current — started before the installed code was last written ⇒

package/src/services/interview-errors.ts

@@ -0,0 +1,37 @@
+/**
+ * The two error types that mean "this interview question was not decided".
+ * Their own module so `responder-probe` (nobody is listening at all) and the
+ * session reaper (a parked question expired) can both throw without importing
+ * each other.
+ *
+ * Callers catch these to distinguish a question that was never decided from one
+ * that was answered — the distinction `module update` conflated when it reported
+ * a breaking update as "operator declined" that no operator had ever seen. The
+ * two are not interchangeable: *unanswered* means no responder could even be
+ * found, *abandoned* means a responder existed, the question stood, and the
+ * deadline passed with nobody deciding.
+ */
+export class InterviewUnansweredError extends Error {
+  constructor(
+    /** The interview event type that went unanswered, e.g. `interview.required.<scope>.<key>`. */
+    readonly queryType: string,
+    message: string,
+  ) {
+    super(message);
+    this.name = 'InterviewUnansweredError';
+  }
+}
+
+/**
+ * The question was posted, stood unanswered past its session's TTL, and the
+ * reaper answered it `abandoned` to release what the parked command held.
+ */
+export class InterviewAbandonedError extends Error {
+  constructor(
+    readonly queryType: string,
+    message: string,
+  ) {
+    super(message);
+    this.name = 'InterviewAbandonedError';
+  }
+}

package/src/services/remote-responder.test.ts

@@ -76,3 +76,86 @@ test('answers responder.probe with kind daemon', async () => {
     responder.close();
   }
 }, 15_000);
+
+/**
+ * A responder that cannot reach a decider must emit NOTHING (celilo#609).
+ *
+ * PR #607 had it reply `{error}` here, which reads as "fail loudly" but carries
+ * `replyFor: <query id>` — it *consumes the query*. The question then no longer
+ * exists for anyone else to answer, and the command dies with it. Leaving it
+ * unanswered parks the command instead, so a later responder can still decide.
+ *
+ * Asserted on the value that comes back, not on liveness: the reply emitted
+ * afterwards correlates to the original query, which is only true if the
+ * responder left it standing.
+ */
+test('an ask that cannot be answered emits no reply — the query stays answerable', async () => {
+  const responder = startRemoteResponder({
+    busDbPath,
+    ask: async () => {
+      throw new Error("stdin isn't a terminal and no answer was pre-staged");
+    },
+  });
+
+  const bus = openBus({ dbPath: busDbPath, events: NO_SCHEMAS });
+  const type = 'interview.required.module-upgrade:iptables.apply_breaking';
+  try {
+    const query = bus.emitRaw(type, {
+      scope: 'module-upgrade:iptables',
+      key: 'apply_breaking',
+      kind: 'confirm',
+      message: 'Apply breaking update for iptables?',
+      required: true,
+      defaultValue: 'false',
+    });
+
+    // Well past the responder's watch latency: still nobody has answered.
+    await new Promise((r) => setTimeout(r, 600));
+    expect(bus.recentEvents({ type: `${type}.reply` })).toHaveLength(0);
+
+    // And the query is still live: a reply emitted now is a genuine answer.
+    bus.emitRaw(
+      `${type}.reply`,
+      { value: false },
+      { replyFor: query.id, emittedBy: 'later-responder' },
+    );
+    const replies = bus.recentEvents({ type: `${type}.reply` });
+    expect(replies).toHaveLength(1);
+    expect((replies[0].payload as { value: unknown }).value).toBe(false);
+    expect(replies[0].replyFor).toBe(query.id);
+  } finally {
+    bus.close();
+    responder.close();
+  }
+}, 15_000);
+
+test('forwards the question scope/key so a client can pre-stage an answer', async () => {
+  const asked: WireInterview[] = [];
+  const responder = startRemoteResponder({
+    busDbPath,
+    ask: async (iv) => {
+      asked.push(iv);
+      return true;
+    },
+  });
+
+  const bus = openBus({ dbPath: busDbPath, events: NO_SCHEMAS });
+  try {
+    await bus.query(
+      'interview.required.module-upgrade:iptables.apply_breaking' as never,
+      {
+        scope: 'module-upgrade:iptables',
+        key: 'apply_breaking',
+        kind: 'confirm',
+        message: 'Apply breaking update for iptables?',
+        required: true,
+      } as never,
+      { timeoutMs: 8000, pollIntervalMs: 100, expect: 'first' } as never,
+    );
+    expect(asked[0].scope).toBe('module-upgrade:iptables');
+    expect(asked[0].key).toBe('apply_breaking');
+  } finally {
+    bus.close();
+    responder.close();
+  }
+}, 15_000);

package/src/services/remote-responder.ts

@@ -20,6 +20,9 @@ const NO_SCHEMAS = defineEvents({});
 /** The normalized question handed to the wire (mirrors InterviewMessage). */
 export interface WireInterview {
   id: string;
+  /** The question's stable identity, so a client can pre-stage `<scope>.<key>`. */
+  scope: string;
+  key: string;
   kind: InterviewRequiredPayload['kind'];
   message: string;
   description?: string;
@@ -54,16 +57,34 @@ export function startRemoteResponder(opts: RemoteResponderOptions): RemoteRespon
       return;
     }
 
-    const value = await opts.ask({
-      id: String(event.id),
-      kind: payload.kind,
-      message: payload.message,
-      description: payload.description,
-      defaultValue: payload.defaultValue,
-      placeholder: payload.placeholder,
-      options: payload.options,
-      required: payload.required,
-    });
+    let value: unknown;
+    try {
+      value = await opts.ask({
+        id: String(event.id),
+        scope: payload.scope,
+        key: payload.key,
+        kind: payload.kind,
+        message: payload.message,
+        description: payload.description,
+        defaultValue: payload.defaultValue,
+        placeholder: payload.placeholder,
+        options: payload.options,
+        required: payload.required,
+      });
+    } catch (err) {
+      // The client had no way to answer — so we emit NOTHING. A reply of any
+      // shape consumes the query (it carries `replyFor: event.id`), destroying
+      // a question nobody has answered yet; the asking command then dies with
+      // it and no other responder can ever act. Leaving it unanswered parks the
+      // command instead, which is what `busInterview`'s `timeoutMs: 0` is for.
+      // The client learns it is parked from the `blocked` wire message.
+      process.stderr.write(
+        `[remote-responder] parked ${event.type} (#${event.id}): ${
+          err instanceof Error ? err.message : String(err)
+        }\n`,
+      );
+      return;
+    }
 
     bus.emitRaw(`${event.type}.reply`, { value }, { replyFor: event.id, emittedBy: me });
   });

package/src/services/responder-probe.ts

@@ -14,6 +14,7 @@
 
 import { defineEvents, openBus } from '@celilo/event-bus';
 import { getEventBusPath } from '../config/paths';
+import { InterviewUnansweredError } from './interview-errors';
 
 const NO_SCHEMAS = defineEvents({});
 
@@ -63,7 +64,8 @@ export async function ensureResponderForInterview(queryType: string): Promise<vo
   if (process.stdin.isTTY) return;
   const available = await probeForResponder(getEventBusPath());
   if (available) return;
-  throw new Error(
+  throw new InterviewUnansweredError(
+    queryType,
     `No responder is listening and stdin isn't a TTY, so this interview prompt can't be answered (${queryType}).
 
 Either: