@celilo/cli 4.0.0 → 5.0.0

package/src/services/system-reboot.test.ts

@@ -0,0 +1,231 @@
+/**
+ * The recurrence gate celilo#1378 asks for: drive the reboot against a FAKE
+ * host, with an operation in flight (it must refuse and name it) and with
+ * nothing in flight (it must reboot, and the return must be verified rather
+ * than assumed).
+ *
+ * Every dependency is injected, so nothing here reboots anything.
+ */
+
+import { describe, expect, it } from 'bun:test';
+import type { InFlightConflict } from './module-operations';
+import {
+  type RebootMarker,
+  collectRebootPendingFacts,
+  judgeRebootPending,
+  requestReboot,
+  verifyReturn,
+} from './system-reboot';
+
+const BOOT_A = '11111111-1111-1111-1111-111111111111';
+const BOOT_B = '22222222-2222-2222-2222-222222222222';
+
+function conflict(describe: string): InFlightConflict {
+  return { operation: {} as InFlightConflict['operation'], describe };
+}
+
+/** A fake host: records what was asked of it, answers what it was told to. */
+function fakeHost(opts: { conflicts?: string[]; bootId?: string | null } = {}) {
+  const state = { rebooted: 0, marker: null as RebootMarker | null };
+  return {
+    state,
+    deps: {
+      conflicts: () => (opts.conflicts ?? []).map(conflict),
+      bootId: () => (opts.bootId === undefined ? BOOT_A : opts.bootId),
+      issueReboot: () => {
+        state.rebooted += 1;
+        return { status: 0 };
+      },
+      writeMarker: (m: RebootMarker) => {
+        state.marker = m;
+      },
+      now: () => 1_700_000_000_000,
+    },
+  };
+}
+
+describe('requestReboot — refuses while the fleet is busy', () => {
+  it('refuses with an operation in flight, and names which one', () => {
+    const host = fakeHost({ conflicts: ['deploy of homebridge (pid 12345)'] });
+
+    const result = requestReboot(host.deps, 'pending kernel');
+
+    expect(result.success).toBe(false);
+    expect(result.error).toContain('deploy of homebridge (pid 12345)');
+    // The whole point: naming it is not enough if it rebooted anyway.
+    expect(host.state.rebooted).toBe(0);
+    expect(host.state.marker).toBe(null);
+  });
+
+  it('refuses when the boot id cannot be read, because the return could never be verified', () => {
+    const host = fakeHost({ bootId: null });
+
+    const result = requestReboot(host.deps);
+
+    expect(result.success).toBe(false);
+    expect(result.error).toContain('could not be verified');
+    expect(host.state.rebooted).toBe(0);
+  });
+
+  it('reboots with nothing in flight, recording the boot id the return is checked against', () => {
+    const host = fakeHost();
+
+    const result = requestReboot(host.deps, 'pending kernel 6.8.0-1064-raspi');
+
+    expect(result.success).toBe(true);
+    expect(host.state.rebooted).toBe(1);
+    expect(host.state.marker?.bootIdBefore).toBe(BOOT_A);
+    expect(host.state.marker?.reason).toContain('6.8.0-1064-raspi');
+    // It reports the reboot as ISSUED, never as completed — the process
+    // asking for it dies in it (celilo#1378).
+    expect(result.message).toContain('--verify');
+  });
+
+  it('leaves the marker in place when the reboot command itself fails', () => {
+    const host = fakeHost();
+    const result = requestReboot(
+      { ...host.deps, issueReboot: () => ({ status: 1 }) },
+      'pending kernel',
+    );
+
+    expect(result.success).toBe(false);
+    expect(result.error).toContain('was not rebooted');
+  });
+});
+
+describe('verifyReturn — only the host actually coming back is success', () => {
+  const marker: RebootMarker = { bootIdBefore: BOOT_A, requestedAt: 1, reason: 'kernel' };
+  const cameBack = {
+    bootId: () => BOOT_B,
+    readMarker: () => marker,
+    heartbeatLive: async () => true,
+    cliVersion: () => '3.0.1',
+  };
+
+  it('reports success once the boot id changed, the dispatcher is live and the CLI answers', async () => {
+    let cleared = 0;
+    const result = await verifyReturn({
+      ...cameBack,
+      clearMarker: () => {
+        cleared += 1;
+      },
+    });
+
+    expect(result.success).toBe(true);
+    expect(result.message).toContain('3.0.1');
+    expect(cleared).toBe(1);
+  });
+
+  it('fails when the boot id is unchanged — the host never went down', async () => {
+    const result = await verifyReturn({ ...cameBack, bootId: () => BOOT_A, clearMarker: () => {} });
+
+    expect(result.success).toBe(false);
+    expect(result.error).toContain('has NOT rebooted');
+  });
+
+  it('fails when the box booted but the dispatcher did not, and says so', async () => {
+    const result = await verifyReturn({
+      ...cameBack,
+      heartbeatLive: async () => false,
+      clearMarker: () => {},
+    });
+
+    expect(result.success).toBe(false);
+    expect(result.error).toContain('dispatcher');
+    expect(result.error).toContain('unverified');
+  });
+
+  it('fails when the CLI does not answer', async () => {
+    const result = await verifyReturn({
+      ...cameBack,
+      cliVersion: () => null,
+      clearMarker: () => {},
+    });
+
+    expect(result.success).toBe(false);
+    expect(result.error).toContain('--version');
+  });
+
+  it('keeps the marker when verification fails, so the audit still sees it', async () => {
+    let cleared = 0;
+    await verifyReturn({
+      ...cameBack,
+      heartbeatLive: async () => false,
+      clearMarker: () => {
+        cleared += 1;
+      },
+    });
+
+    expect(cleared).toBe(0);
+  });
+
+  it('says so plainly when no reboot was ever requested', async () => {
+    const result = await verifyReturn({ ...cameBack, readMarker: () => null });
+
+    expect(result.success).toBe(false);
+    expect(result.error).toContain('No reboot is pending');
+  });
+});
+
+describe('judgeRebootPending — two independent tells', () => {
+  const none = {
+    markerPresent: false,
+    deferredPackages: [],
+    runningKernel: '6.8.0-1064-raspi',
+    installedKernel: '6.8.0-1064-raspi',
+  };
+
+  it('is the #1378 case: a running kernel older than the installed one', () => {
+    const verdict = judgeRebootPending({ ...none, runningKernel: '6.8.0-1063-raspi' });
+
+    expect(verdict.pending).toBe(true);
+    expect(verdict.reasons[0]).toContain('6.8.0-1063-raspi');
+    expect(verdict.reasons[0]).toContain('6.8.0-1064-raspi');
+  });
+
+  it("reports Debian's own marker, naming the packages that deferred", () => {
+    const verdict = judgeRebootPending({
+      ...none,
+      markerPresent: true,
+      deferredPackages: ['docker.service', 'dbus.service'],
+    });
+
+    expect(verdict.pending).toBe(true);
+    expect(verdict.reasons.join(' ')).toContain('docker.service');
+  });
+
+  it('is clean when the kernel matches and nothing deferred', () => {
+    expect(judgeRebootPending(none)).toEqual({ pending: false, unmeasured: false, reasons: [] });
+  });
+
+  it('is UNMEASURED, not clean, when the host could not be asked', () => {
+    // An absent /var/run/reboot-required on a host that has no /var/run is not
+    // evidence of a healthy kernel. Reporting it clean is the #1326 shape.
+    const verdict = judgeRebootPending({
+      markerPresent: false,
+      deferredPackages: [],
+      runningKernel: null,
+      installedKernel: null,
+    });
+
+    expect(verdict).toEqual({ pending: false, unmeasured: true, reasons: [] });
+  });
+});
+
+describe('collectRebootPendingFacts', () => {
+  it('reads the marker and splits the deferred package list', () => {
+    const facts = collectRebootPendingFacts(
+      (path) => (path.endsWith('.pkgs') ? 'docker\n\ndbus\n' : ''),
+      '6.8.0-1063-raspi',
+      '6.8.0-1064-raspi',
+    );
+
+    expect(facts.markerPresent).toBe(true);
+    expect(facts.deferredPackages).toEqual(['docker', 'dbus']);
+    expect(facts.runningKernel).toBe('6.8.0-1063-raspi');
+  });
+
+  it('reports no marker when the file is absent', () => {
+    expect(collectRebootPendingFacts(() => null).markerPresent).toBe(false);
+  });
+});

package/src/services/system-reboot.ts

@@ -0,0 +1,357 @@
+/**
+ * Rebooting the management server (celilo#1378).
+ *
+ * celilo already DETECTS that celilo-mgr needs a reboot — `apt-upgrade` prints
+ * needrestart's pending-kernel block — and until now offered no way to act on
+ * it. The only path was SSH and `sudo reboot`: out of band, unrecorded, and not
+ * something an agent can reach, since agents operate celilo-mgr only through
+ * the MCP server.
+ *
+ * Rebooting this box is not `systemctl reboot`. It takes down the event
+ * dispatcher, the MCP server, the remote API and anything deploying, so the
+ * reboot is gated on nothing being in flight and on explicit consent.
+ *
+ * WHY VERIFICATION IS A SECOND CALL. The process that asks for the reboot dies
+ * in it — it runs ON the host it is rebooting. So no single invocation can both
+ * reboot and report that the host came back; a command that claimed to would be
+ * reporting a request as an outcome, which is the thing the issue asks us not
+ * to do. Instead `requestReboot` records a marker carrying the CURRENT boot id,
+ * and `verifyReturn` is what reports success: it passes only once the boot id
+ * has actually changed AND the dispatcher and CLI answer. An unverified marker
+ * is a reboot that did not come back, and the audit reports it as one.
+ *
+ * WHAT "IN FLIGHT" COVERS, and what it does not. The gate is `checkInFlight`,
+ * the module-operation lock that backup and restore already consult — a deploy,
+ * uninstall, backup or restore registers there, so all four hold the reboot off.
+ * celilo#1378 also names the e2e rig lock; that one is deliberately NOT checked,
+ * because the rig runs on a developer's machine and never on celilo-mgr, so
+ * reading it here would be a check that cannot reach its subject.
+ *
+ * Every judgement here is pure over injected facts, so the refusal and the
+ * verification are both testable against a fake host.
+ */
+
+import { mkdirSync, readFileSync, readdirSync, rmSync, writeFileSync } from 'node:fs';
+import { release } from 'node:os';
+import { dirname, join } from 'node:path';
+import { getDataDir } from '../config/paths';
+import { type InFlightConflict, checkInFlight } from './module-operations';
+
+/** Debian writes this the moment a package needs a reboot to take effect. */
+export const REBOOT_REQUIRED_PATH = '/var/run/reboot-required';
+/** …and the packages that asked for it here, one per line. */
+export const REBOOT_REQUIRED_PKGS_PATH = '/var/run/reboot-required.pkgs';
+/** The kernel's own identity for this boot. It changes on every boot, and only then. */
+export const BOOT_ID_PATH = '/proc/sys/kernel/random/boot_id';
+
+/** What the host says about whether it needs a reboot. */
+export interface RebootPendingFacts {
+  /** `/var/run/reboot-required` is present. */
+  markerPresent: boolean;
+  /** Packages named in `/var/run/reboot-required.pkgs`. */
+  deferredPackages: string[];
+  /** `uname -r`, or null when it could not be read. */
+  runningKernel: string | null;
+  /** The newest kernel installed on disk, or null when it could not be read. */
+  installedKernel: string | null;
+}
+
+export interface RebootPendingVerdict {
+  pending: boolean;
+  /** True when the host could not be asked at all — NOT the same as "no reboot needed". */
+  unmeasured: boolean;
+  /** Operator-readable reasons, in the order they were found. */
+  reasons: string[];
+}
+
+/**
+ * Two independent tells, and either one is enough.
+ *
+ * The marker file is Debian's own flag and covers the deferred service
+ * restarts. The kernel comparison covers the case the marker misses and the
+ * case #1378 was filed from: running 6.8.0-1063-raspi with 1064 installed.
+ */
+export function judgeRebootPending(facts: RebootPendingFacts): RebootPendingVerdict {
+  const { markerPresent, deferredPackages, runningKernel, installedKernel } = facts;
+
+  const kernelKnown = runningKernel !== null && installedKernel !== null;
+  // Nothing was measured: no marker AND no kernel pair to compare. An absent
+  // marker alone is not evidence of a healthy host — the file lives under
+  // /var/run, which a container or a non-Debian host simply does not have.
+  if (!markerPresent && !kernelKnown) {
+    return { pending: false, unmeasured: true, reasons: [] };
+  }
+
+  const reasons: string[] = [];
+  if (kernelKnown && runningKernel !== installedKernel) {
+    reasons.push(
+      `the running kernel is ${runningKernel} but ${installedKernel} is installed — the new kernel loads on reboot and not before`,
+    );
+  }
+  if (markerPresent) {
+    reasons.push(
+      deferredPackages.length > 0
+        ? `${REBOOT_REQUIRED_PATH} is present; deferred by: ${deferredPackages.join(', ')}`
+        : `${REBOOT_REQUIRED_PATH} is present`,
+    );
+  }
+
+  return { pending: reasons.length > 0, unmeasured: false, reasons };
+}
+
+/** Read the host's own answer. The only impure part of the detection. */
+export function collectRebootPendingFacts(
+  read: (path: string) => string | null = readFileOrNull,
+  runningKernel: string | null = null,
+  installedKernel: string | null = null,
+): RebootPendingFacts {
+  return {
+    markerPresent: read(REBOOT_REQUIRED_PATH) !== null,
+    deferredPackages: (read(REBOOT_REQUIRED_PKGS_PATH) ?? '')
+      .split('\n')
+      .map((line) => line.trim())
+      .filter((line) => line.length > 0),
+    runningKernel,
+    installedKernel,
+  };
+}
+
+function readFileOrNull(path: string): string | null {
+  try {
+    return readFileSync(path, 'utf-8');
+  } catch {
+    return null;
+  }
+}
+
+/** The marker a reboot request leaves behind, so the return can be verified. */
+export interface RebootMarker {
+  /** The boot id at the moment the reboot was requested. */
+  bootIdBefore: string;
+  /** Epoch ms. */
+  requestedAt: number;
+  /** Why the operator asked — echoed back by the audit finding. */
+  reason: string;
+}
+
+export function rebootMarkerPath(dataDir: string = getDataDir()): string {
+  return join(dataDir, 'state', 'reboot-requested.json');
+}
+
+export function readRebootMarker(path: string = rebootMarkerPath()): RebootMarker | null {
+  try {
+    const parsed = JSON.parse(readFileSync(path, 'utf-8')) as Partial<RebootMarker>;
+    if (typeof parsed.bootIdBefore !== 'string' || typeof parsed.requestedAt !== 'number') {
+      return null;
+    }
+    return {
+      bootIdBefore: parsed.bootIdBefore,
+      requestedAt: parsed.requestedAt,
+      reason: parsed.reason ?? '',
+    };
+  } catch {
+    return null;
+  }
+}
+
+export function writeRebootMarker(marker: RebootMarker, path: string = rebootMarkerPath()): void {
+  mkdirSync(dirname(path), { recursive: true });
+  writeFileSync(path, `${JSON.stringify(marker, null, 2)}\n`, 'utf-8');
+}
+
+export function clearRebootMarker(path: string = rebootMarkerPath()): void {
+  rmSync(path, { force: true });
+}
+
+export interface RequestRebootDeps {
+  /** In-flight module operations. Defaults to the real `checkInFlight`. */
+  conflicts?: () => InFlightConflict[];
+  /** This boot's id, or null when unreadable. */
+  bootId: () => string | null;
+  /** Fire the reboot. Only reached once every gate above has passed. */
+  issueReboot: () => { status: number | null };
+  writeMarker?: (marker: RebootMarker) => void;
+  now?: () => number;
+}
+
+export interface RequestRebootResult {
+  success: boolean;
+  error?: string;
+  message?: string;
+  /** The boot id recorded for `verifyReturn` to compare against. */
+  bootIdBefore?: string;
+}
+
+/**
+ * Refuse, or reboot. Consent is the CALLER's to obtain (flag or interview) —
+ * this function is the part that must not be bypassed, so it holds only the
+ * checks, never the prompt.
+ */
+export function requestReboot(deps: RequestRebootDeps, reason = ''): RequestRebootResult {
+  const conflicts = (deps.conflicts ?? checkInFlight)();
+  if (conflicts.length > 0) {
+    // Name what we are waiting on. "Busy, try later" sends the operator to
+    // guess, and the guess is usually a reboot anyway.
+    return {
+      success: false,
+      error: `Refusing to reboot: ${conflicts.length} module operation(s) in flight.\n${conflicts
+        .map((c) => `  • ${c.describe}`)
+        .join(
+          '\n',
+        )}\nWait for them to finish, or release them with \`celilo module operations clear\` if they are not really running.`,
+    };
+  }
+
+  const bootIdBefore = deps.bootId();
+  if (bootIdBefore === null) {
+    // Without a before-value there is nothing to compare after, so a reboot
+    // here could never be verified — and an unverifiable reboot of the
+    // control plane is exactly what this command exists to replace.
+    return {
+      success: false,
+      error: `Refusing to reboot: could not read this boot's id from ${BOOT_ID_PATH}, so the host coming back could not be verified afterwards.`,
+    };
+  }
+
+  (deps.writeMarker ?? writeRebootMarker)({
+    bootIdBefore,
+    requestedAt: (deps.now ?? Date.now)(),
+    reason,
+  });
+
+  const { status } = deps.issueReboot();
+  if (status !== 0) {
+    return {
+      success: false,
+      error: `Reboot command exited ${status ?? 'on a signal'}. The host was not rebooted; the pending marker is left in place for \`celilo system reboot --verify\` to clear.`,
+    };
+  }
+
+  return {
+    success: true,
+    bootIdBefore,
+    message:
+      'Reboot issued. This process goes down with the host, so it cannot report the return — ' +
+      'run `celilo system reboot --verify` once celilo-mgr answers again, or read `celilo system audit`, ' +
+      'which reports an unverified reboot as a finding.',
+  };
+}
+
+export interface VerifyReturnDeps {
+  bootId: () => string | null;
+  readMarker?: () => RebootMarker | null;
+  clearMarker?: () => void;
+  /** Is the event dispatcher alive and emitting? */
+  heartbeatLive: () => Promise<boolean>;
+  /** Does the installed CLI answer `--version`? Returns the version, or null. */
+  cliVersion: () => string | null;
+}
+
+export interface VerifyReturnResult {
+  success: boolean;
+  error?: string;
+  message?: string;
+}
+
+/**
+ * The half that is allowed to say "the host came back".
+ *
+ * All three must hold: the boot id CHANGED (so we are not reading the machine
+ * that never went down), the dispatcher is live, and the CLI answers. Any one
+ * of them alone passes on a host that is half up — a booted box whose
+ * dispatcher never started is not a successful reboot of a control plane.
+ */
+export async function verifyReturn(deps: VerifyReturnDeps): Promise<VerifyReturnResult> {
+  const marker = (deps.readMarker ?? (() => readRebootMarker()))();
+  if (marker === null) {
+    return {
+      success: false,
+      error: 'No reboot is pending verification — nothing recorded a reboot request.',
+    };
+  }
+
+  const bootIdNow = deps.bootId();
+  if (bootIdNow === null) {
+    return {
+      success: false,
+      error: `Could not read ${BOOT_ID_PATH}, so the reboot could not be verified.`,
+    };
+  }
+  if (bootIdNow === marker.bootIdBefore) {
+    return {
+      success: false,
+      error: `The host has NOT rebooted: the boot id is still ${bootIdNow}, the one recorded when the reboot was requested.`,
+    };
+  }
+
+  const heartbeat = await deps.heartbeatLive();
+  const version = deps.cliVersion();
+  if (!heartbeat || version === null) {
+    const missing = [
+      heartbeat ? null : 'the event dispatcher is not reporting a live heartbeat',
+      version === null ? 'the installed celilo CLI did not answer `--version`' : null,
+    ].filter((m): m is string => m !== null);
+    return {
+      success: false,
+      error: `The host rebooted but did not come back fully: ${missing.join('; ')}. The reboot stays unverified.`,
+    };
+  }
+
+  (deps.clearMarker ?? (() => clearRebootMarker()))();
+  return {
+    success: true,
+    message: `celilo-mgr rebooted and came back: dispatcher live, CLI ${version} answering.`,
+  };
+}
+
+/**
+ * The newest kernel the host has on disk, from /boot/vmlinuz-<release>.
+ *
+ * Compared against `os.release()` — the kernel actually running — because
+ * that pair is the #1378 condition itself. Sorted numerically per dotted
+ * component so `-1064-` beats `-999-`, which a plain string sort gets wrong.
+ */
+export function newestInstalledKernel(bootEntries: string[], prefix = 'vmlinuz-'): string | null {
+  const releases = bootEntries
+    .filter((name) => name.startsWith(prefix))
+    .map((name) => name.slice(prefix.length))
+    .filter((release) => release.length > 0);
+  if (releases.length === 0) return null;
+
+  return releases.sort(compareKernelRelease).at(-1) ?? null;
+}
+
+function compareKernelRelease(a: string, b: string): number {
+  const partsOf = (s: string) => s.split(/[.\-+]/);
+  const pa = partsOf(a);
+  const pb = partsOf(b);
+  for (let i = 0; i < Math.max(pa.length, pb.length); i++) {
+    const na = Number(pa[i]);
+    const nb = Number(pb[i]);
+    const bothNumeric = Number.isFinite(na) && Number.isFinite(nb);
+    const cmp = bothNumeric ? na - nb : (pa[i] ?? '').localeCompare(pb[i] ?? '');
+    if (cmp !== 0) return cmp;
+  }
+  return 0;
+}
+
+/** Everything the audit check needs, read from this host. */
+export function collectRebootPendingDeps(
+  bootDir = '/boot',
+  markerPath: string = rebootMarkerPath(),
+): { verdict: RebootPendingVerdict; marker: RebootMarker | null } {
+  let bootEntries: string[] = [];
+  try {
+    bootEntries = readdirSync(bootDir);
+  } catch {
+    // No /boot to read — the kernel half is simply unmeasured, which
+    // judgeRebootPending reports as such rather than as a clean host.
+  }
+
+  return {
+    verdict: judgeRebootPending(
+      collectRebootPendingFacts(readFileOrNull, release(), newestInstalledKernel(bootEntries)),
+    ),
+    marker: readRebootMarker(markerPath),
+  };
+}

package/src/services/update/dep-graph.test.ts

@@ -20,7 +20,7 @@ function makeManifest(
     id,
     name: id,
     version: '1.0.0',
-    celilo_contract: '1.0',
+    celilo_contract: '2.0',
     provides: opts.provides
       ? {
           capabilities: opts.provides.map((name) => ({