@celilo/cli 4.0.0 → 4.1.0

package/src/cli/tui/audit-state.ts

@@ -73,6 +73,8 @@ export const ALL_CATEGORIES: readonly DriftCategory[] = [
   'schema',
   'capability_abi',
   'browser_pin',
+  'server_bun_pin',
+  'reboot_pending',
   'terraform_plan',
   'module_versions',
   'module_configs',
@@ -96,6 +98,8 @@ export const ALL_CATEGORIES: readonly DriftCategory[] = [
 ];
 
 export const CATEGORY_LABELS: Record<DriftCategory, string> = {
+  server_bun_pin: 'Server bun pin',
+  reboot_pending: 'Pending reboot',
   module_integrity: 'Module integrity',
   detect_without_converge: 'Drift without converge',
   jail_exemptions: 'Hook jail exemptions',

package/src/hooks/broker.ts

@@ -105,6 +105,18 @@ export interface Broker {
   outcome(): HookOutcome | undefined;
   /** Every protocol-level complaint, for the error message when one matters. */
   faults(): string[];
+  /**
+   * The capability calls that are still in flight, newest last, each rendered
+   * as `provider_module:capability.method (Ns)`.
+   *
+   * A hook that dies at its total timeout has usually died WAITING on one of
+   * these, and the bare "Hook total timeout exceeded" names only the hook —
+   * never what it was waiting for. Measured 2026-09-21 (celilo#1403):
+   * bna-yard-sale's on_install died at 120s blocked on caddy's 75s route
+   * reconcile, and the log's only tell was a `publishStaticSite` push line
+   * with no matching completion. The operator had to infer the provider.
+   */
+  pendingCalls(): string[];
   /** Refuse further calls. Idempotent. Called on kill and again on cleanup. */
   stop(): void;
   /** Close the socket and remove its directory. */
@@ -184,6 +196,8 @@ export async function startBroker(options: BrokerOptions): Promise<Broker> {
   let outcome: HookOutcome | undefined;
   let stopped = false;
   const faults: string[] = [];
+  /** In-flight capability calls by frame id, so a timeout can name them. */
+  const pending = new Map<string, { capability: string; method: string; startedAt: number }>();
   let connection: Socket | undefined;
   // Already resolved: no connection means nothing left to deliver.
   let connectionClosed: Promise<void> = Promise.resolve();
@@ -300,6 +314,7 @@ export async function startBroker(options: BrokerOptions): Promise<Broker> {
       return;
     }
 
+    pending.set(id, { capability: capabilityName, method, startedAt: Date.now() });
     try {
       const value = await (fn as (...a: unknown[]) => unknown).apply(capability, args);
       options.onActivity();
@@ -307,6 +322,8 @@ export async function startBroker(options: BrokerOptions): Promise<Broker> {
     } catch (error) {
       options.onActivity();
       send({ type: 'throw', id, error: serializeError(error) });
+    } finally {
+      pending.delete(id);
     }
   }
 
@@ -384,6 +401,20 @@ export async function startBroker(options: BrokerOptions): Promise<Broker> {
       ]),
     outcome: () => outcome,
     faults: () => [...faults],
+    pendingCalls: () => {
+      const now = Date.now();
+      return [...pending.values()]
+        .sort((a, b) => a.startedAt - b.startedAt)
+        .map((call) => {
+          // `providerModuleId` is stamped onto the capability's data half by
+          // the loader, so the message can name the MODULE the hook is waiting
+          // on and not only the capability it asked through.
+          const view = options.capabilities[call.capability] as Record<string, unknown> | undefined;
+          const provider = typeof view?.providerModuleId === 'string' ? view.providerModuleId : '';
+          const waited = Math.round((now - call.startedAt) / 1000);
+          return `${provider ? `${provider}:` : ''}${call.capability}.${call.method} (${waited}s)`;
+        });
+    },
     stop: () => {
       stopped = true;
     },

package/src/hooks/capability-loader.ts

@@ -103,14 +103,20 @@ async function requireProviderReconcile(consumingModuleId: string, what: string)
       `public_web ${what} for ${consumingModuleId} was persisted but NOT delivered to the provider (caddy): no event dispatcher is running, so caddy never reconciled and the hostname has no site block or cert. Run this through \`celilo module deploy\` (which runs the dispatcher) rather than a bare hook.`,
     );
   }
+  // The handler's own words, when it left any. A crash is retried, so its
+  // delivery sits in `pending` with the error already recorded while the
+  // deadline runs out — which is how celilo#1401 reported a TIMEOUT for a
+  // caddy crash, for days, with `0 ok, 0 failed of 1` and the cause nowhere.
+  const because = reconcile.lastError ? ` The provider reported: ${reconcile.lastError}` : '';
+
   if (reconcile.timedOut) {
     throw new Error(
-      `public_web reconcile for ${consumingModuleId} (${what}) did not finish within the deadline (${reconcile.succeeded} ok, ${reconcile.failed} failed of ${reconcile.events} change event(s)) — caddy did not confirm the change is live.`,
+      `public_web reconcile for ${consumingModuleId} (${what}) did not finish within the deadline (${reconcile.succeeded} ok, ${reconcile.failed} failed of ${reconcile.events} change event(s)) — caddy did not confirm the change is live.${because}`,
     );
   }
   if (reconcile.failed > 0) {
     throw new Error(
-      `public_web reconcile for ${consumingModuleId} (${what}): ${reconcile.failed} delivery(ies) failed — caddy could not apply the change, so the hostname is not served as declared.`,
+      `public_web reconcile for ${consumingModuleId} (${what}): ${reconcile.failed} delivery(ies) failed — caddy could not apply the change, so the hostname is not served as declared.${because}`,
     );
   }
   // ISS-0087: a change WAS persisted and the dispatcher IS alive, yet ZERO

package/src/hooks/executor.ts

@@ -511,7 +511,14 @@ export async function executeHookScript(
       setTimeout(() => child.kill('SIGKILL'), SIGKILL_GRACE_MS).unref();
     };
 
-    const totalTimer = setTimeout(() => kill('total'), timeoutMs);
+    // Snapshotted inside the timer rather than read at throw time: the child's
+    // death rejects its in-flight calls, so by the time we build the message
+    // the map that knows the answer has drained itself.
+    let pendingCallsAtKill: string[] = [];
+    const totalTimer = setTimeout(() => {
+      pendingCallsAtKill = broker.pendingCalls();
+      kill('total');
+    }, timeoutMs);
     // Debug runs get no idle timer, same as before: an operator stepping
     // through a browser hook is silent for minutes on purpose.
     const idleTimer = context.debug
@@ -536,7 +543,18 @@ export async function executeHookScript(
     // nothing keeps running past this line. That is the difference from the
     // `Promise.race` this replaces (celilo#1003).
     if (killedFor === 'total') {
-      throw new Error(`Hook total timeout exceeded (${Math.round(timeoutMs / 1000)}s)`);
+      // Name what it was WAITING ON, not just that it waited. A hook whose
+      // budget is consumed by one slow provider used to die as a bare
+      // "Hook total timeout exceeded", so the operator had to infer the
+      // provider from a capability push line with no matching completion
+      // (celilo#1403: bna-yard-sale's 120s on_install blocked on caddy's 75s
+      // route reconcile). Captured before `close()`, while the calls are
+      // still in flight — `stop()` refuses new calls but clears nothing.
+      const waitingOn = pendingCallsAtKill;
+      const waitingSuffix = waitingOn.length > 0 ? `, waiting on ${waitingOn.join(', ')}` : '';
+      throw new Error(
+        `Hook total timeout exceeded (${Math.round(timeoutMs / 1000)}s)${waitingSuffix}`,
+      );
     }
     if (killedFor === 'idle') {
       throw new Error(

package/src/hooks/test-fixtures/slow-capability-hook.ts

@@ -0,0 +1,28 @@
+/**
+ * Test fixture: a hook that blocks inside a capability call it never gets an
+ * answer to, and is killed at its own total timeout (celilo#1403).
+ *
+ * The real shape is a consumer's `on_install` calling
+ * `public_web.publishStaticSite`, which blocks on the provider's route
+ * reconcile. This fixture reproduces the boundary condition — a call in flight
+ * when the kill lands — without a provider.
+ *
+ * It logs BEFORE the call so the run is not mistaken for an idle timeout, then
+ * awaits forever.
+ */
+
+import { defineHook } from '@celilo/capabilities';
+
+interface SlowCapability {
+  neverReturns(request: Record<string, unknown>): Promise<unknown>;
+}
+
+export default defineHook({
+  hook: 'container_created',
+  requires: [],
+  handler: async (ctx) => {
+    const slow = (ctx.capabilities as unknown as Record<string, SlowCapability>).slow_provider;
+    ctx.logger.info('calling the provider');
+    await slow.neverReturns({});
+  },
+});

package/src/hooks/timeout-names-provider.test.ts

@@ -0,0 +1,87 @@
+/**
+ * A hook killed at its TOTAL timeout must name what it was waiting on.
+ *
+ * Measured 2026-09-21 (celilo#1403): bna-yard-sale's `on_install` called
+ * `public_web.publishStaticSite`, which blocks on caddy's route reconcile —
+ * 70-75s across the fleet's sites. The consumer died at its 120s budget with:
+ *
+ *     [bna-yard-sale:on_install] -> public_web.publishStaticSite
+ *     [bna-yard-sale:on_install] OK dns_registrar.registerHost
+ *     [bna-yard-sale:on_install] OK dns_internal.registerRecord
+ *     [bna-yard-sale:on_install] FAIL Hook total timeout exceeded (120s)
+ *
+ * Both DNS calls completed; the push line had no matching completion, and that
+ * ABSENCE was the only evidence of which provider held it. The error named the
+ * hook and never the cause, so the operator had to infer it from a gap in the
+ * log — the same class of missing signal the greenwave/axon ISP-router hooks
+ * already carry a comment about.
+ */
+import { describe, expect, test } from 'bun:test';
+import { mkdtempSync, rmSync } from 'node:fs';
+import { tmpdir } from 'node:os';
+import { join } from 'node:path';
+import { executeHookScript } from './executor';
+import { createCapturingLogger } from './logger';
+import { configStore, secretStore } from './test-fixtures/store-backed';
+import type { HookContext } from './types';
+
+const FIXTURES = join(__dirname, 'test-fixtures');
+
+/** A capability whose one method never settles — the provider that holds the hook. */
+function hangingCapabilities(): Record<string, unknown> {
+  return {
+    slow_provider: {
+      providerModuleId: 'caddy',
+      version: '1.0.0',
+      neverReturns: () => new Promise(() => {}),
+    },
+  };
+}
+
+async function runUntilTotalTimeout(): Promise<Error> {
+  const dir = mkdtempSync(join(tmpdir(), 'celilo-timeout-'));
+  try {
+    const context: HookContext = {
+      config: configStore(),
+      secrets: secretStore(),
+      systems: [],
+      logger: createCapturingLogger().logger,
+      debug: false,
+      screenshotDir: dir,
+      stateDir: dir,
+      capabilities: hangingCapabilities(),
+    };
+    try {
+      await executeHookScript(join(FIXTURES, 'slow-capability-hook.ts'), context, {
+        // Total well under idle, so the kill is unambiguously the TOTAL one:
+        // a hook blocked in a capability call emits nothing, so a shorter idle
+        // timer would win the race and prove a different thing.
+        timeoutMs: 3_000,
+        idleTimeoutMs: 60_000,
+      });
+    } catch (error) {
+      return error as Error;
+    }
+    throw new Error('the hook returned; it was supposed to be killed at its total timeout');
+  } finally {
+    rmSync(dir, { recursive: true, force: true });
+  }
+}
+
+describe('a total-timeout error names the provider it was waiting on (celilo#1403)', () => {
+  test('the run reached the total timeout, not the idle one', async () => {
+    const error = await runUntilTotalTimeout();
+    // Anchor the subject: an idle kill would exercise none of this.
+    expect(error.message).toContain('Hook total timeout exceeded');
+    expect(error.message).not.toContain('idle');
+  }, 20_000);
+
+  test('the message names the provider, the capability and the method', async () => {
+    const error = await runUntilTotalTimeout();
+    expect(error.message).toContain('waiting on');
+    // The provider MODULE is the operationally useful half — it is what the
+    // operator goes and looks at. `caddy` comes from `providerModuleId`.
+    expect(error.message).toContain('caddy');
+    expect(error.message).toContain('slow_provider.neverReturns');
+  }, 20_000);
+});

package/src/services/api-access.test.ts

@@ -3,6 +3,7 @@ import { mkdtempSync, rmSync } from 'node:fs';
 import { tmpdir } from 'node:os';
 import { join } from 'node:path';
 import { closeDb } from '@/db/client';
+import { COMMANDS, flattenLeaves, isReadOnlyPath, opOf, readOnlyGrants } from '@celilo/core';
 import {
   getPrincipalByName,
   grantPrincipal,
@@ -136,3 +137,46 @@ describe('api-access (DB-backed)', () => {
     expect(await listPrincipals()).toHaveLength(0);
   });
 });
+
+/**
+ * The recurrence gate for celilo#1404: every command classified read-only must
+ * actually be reachable by the read-only principal.
+ *
+ * The check is cross-instrument on purpose: `readOnlyGrants` decides what the RO
+ * principal is granted, and `grantsAllow` is the SERVER's own matcher deciding
+ * what that grant admits. What it catches is the two disagreeing about the shape
+ * of an op — which is how a derived list still leaves a read command denied.
+ * Verified by mutation (Rule 7.6): rendering the op as `module.journal` instead of
+ * `module:journal` turns this red.
+ *
+ * What it does NOT catch is the classifier itself calling a read a write — both
+ * sides read the same derivation, so they move together. That direction is
+ * covered in read-only-classifier.test.ts by a hand-written list of ops that must
+ * never be granted.
+ */
+describe('read-only principal reachability (celilo#1404)', () => {
+  const grants = readOnlyGrants(COMMANDS);
+  const readLeaves = flattenLeaves(COMMANDS).filter((l) => isReadOnlyPath(l.path));
+
+  test('the registry has read-only leaves at all', () => {
+    // Otherwise the sweep below is vacuously green.
+    expect(readLeaves.length).toBeGreaterThan(20);
+  });
+
+  test('the server admits every read-only command for the RO principal', () => {
+    const denied = readLeaves
+      .map((l) => ({ path: l.path.join(' '), op: opOf(l.path) }))
+      .filter((l) => !grantsAllow(grants, l.op.split(':')[0], l.op.split(':')[1]));
+    expect(denied).toEqual([]);
+  });
+
+  test('module journal — the op the live server denied with a bare exit 126', () => {
+    expect(grantsAllow(grants, 'module', 'journal')).toBe(true);
+    expect(grantsAllow(grants, 'module', 'where')).toBe(true);
+  });
+
+  test('and still denies a write op', () => {
+    expect(grantsAllow(grants, 'module', 'deploy')).toBe(false);
+    expect(grantsAllow(grants, 'api', 'grant')).toBe(false);
+  });
+});

package/src/services/audit/index.test.ts

@@ -20,6 +20,12 @@ const emptyDeps = {
     db: fakeDb,
   },
   browserPin: { consumers: [], provisioned: null },
+  serverBunPin: {
+    pinnedVersion: '1.4.2',
+    runningVersion: '1.4.2',
+    pinPath: '/etc/celilo/mise.pin.toml',
+  },
+  rebootPending: { verdict: { pending: false, unmeasured: false, reasons: [] }, marker: null },
   capabilityAbi: { modules: [] },
   terraformPlan: {
     modules: [],
@@ -42,6 +48,7 @@ const emptyDeps = {
   secretsDecryptable: { results: [] },
   servicesReachable: { results: [] },
   machinesReachable: { results: [] },
+  machineInterfaceZones: { machines: [] },
   diskSpace: { results: [] },
   transportReads: { statuses: [], now: new Date(), staleAfterMs: 30 * 60_000 },
   trustedSources: { firewalls: [], unreachableFirewalls: [] },

package/src/services/audit/index.ts

@@ -25,13 +25,19 @@ import {
 import { type DiskSpaceAuditDeps, auditDiskSpace } from './disk-space';
 import { type HealthAuditDeps, auditHealth } from './health';
 import { type JailExemptionsAuditDeps, auditJailExemptions } from './jail-exemptions';
+import {
+  type MachineInterfaceZonesAuditDeps,
+  auditMachineInterfaceZones,
+} from './machine-interface-zones';
 import { type MachinesReachableAuditDeps, auditMachinesReachable } from './machines-reachable';
 import { type ModuleConfigsAuditDeps, auditModuleConfigs } from './module-configs';
 import { type ModuleIntegrityAuditDeps, auditModuleIntegrity } from './module-integrity';
 import { type ModuleVersionsAuditDeps, auditModuleVersions } from './module-versions';
 import { type PublicDnsAuditDeps, auditPublicDns } from './public-dns';
+import { type RebootPendingAuditDeps, auditRebootPending } from './reboot-pending';
 import { type SchemaAuditDeps, auditSchema } from './schema';
 import { type SecretsDecryptableAuditDeps, auditSecretsDecryptable } from './secrets-decryptable';
+import { type ServerBunPinAuditDeps, auditServerBunPin } from './server-bun-pin';
 import {
   type ServicesCredentialsAuditDeps,
   auditServicesCredentials,
@@ -57,6 +63,8 @@ export interface AuditDeps {
   schema: SchemaAuditDeps;
   capabilityAbi: CapabilityAbiAuditDeps;
   browserPin: BrowserPinAuditDeps;
+  serverBunPin: ServerBunPinAuditDeps;
+  rebootPending: RebootPendingAuditDeps;
   terraformPlan: TerraformPlanAuditDeps;
   moduleVersions: ModuleVersionsAuditDeps;
   moduleConfigs: ModuleConfigsAuditDeps;
@@ -72,6 +80,7 @@ export interface AuditDeps {
   secretsDecryptable: SecretsDecryptableAuditDeps;
   servicesReachable: ServicesReachableAuditDeps;
   machinesReachable: MachinesReachableAuditDeps;
+  machineInterfaceZones: MachineInterfaceZonesAuditDeps;
   publicDns: PublicDnsAuditDeps;
   diskSpace: DiskSpaceAuditDeps;
   transportReads: TransportReadsAuditDeps;
@@ -115,6 +124,8 @@ export async function runAudit(
     wrap('schema', auditSchema(deps.schema)),
     wrap('capability_abi', auditCapabilityAbi(deps.capabilityAbi)),
     wrap('browser_pin', auditBrowserPin(deps.browserPin)),
+    wrap('server_bun_pin', Promise.resolve(auditServerBunPin(deps.serverBunPin))),
+    wrap('reboot_pending', Promise.resolve(auditRebootPending(deps.rebootPending))),
     wrap('terraform_plan', auditTerraformPlan(deps.terraformPlan)),
     wrap('module_versions', auditModuleVersions(deps.moduleVersions)),
     wrap('module_configs', auditModuleConfigs(deps.moduleConfigs)),
@@ -136,6 +147,10 @@ export async function runAudit(
     wrap('secrets_decryptable', auditSecretsDecryptable(deps.secretsDecryptable)),
     wrap('services_reachable', auditServicesReachable(deps.servicesReachable)),
     wrap('machines_reachable', auditMachinesReachable(deps.machinesReachable)),
+    wrap(
+      'interface_classification',
+      Promise.resolve(auditMachineInterfaceZones(deps.machineInterfaceZones)),
+    ),
     wrap('public_dns', auditPublicDns(deps.publicDns)),
     wrap('disk_space', Promise.resolve(auditDiskSpace(deps.diskSpace))),
     wrap('transport_reads', auditTransportReads(deps.transportReads)),

package/src/services/audit/machine-interface-zones.test.ts

@@ -0,0 +1,30 @@
+import { describe, expect, test } from 'bun:test';
+import { auditMachineInterfaceZones } from './machine-interface-zones';
+
+describe('auditMachineInterfaceZones', () => {
+  test('reports a machine whose stored labels predate the zone model', () => {
+    const findings = auditMachineInterfaceZones({
+      machines: [
+        {
+          hostname: 'celilo-mgr',
+          changes: [
+            { name: 'eth0', ipAddress: '10.0.120.10', from: 'external', to: 'secure-mgmt' },
+          ],
+        },
+      ],
+    });
+
+    expect(findings).toHaveLength(1);
+    expect(findings[0].severity).toBe('drift');
+    expect(findings[0].code).toBe('stale_interface_zone_labels');
+    expect(findings[0].subject).toBe('celilo-mgr');
+    expect(findings[0].message).toContain('external → secure-mgmt');
+    expect(findings[0].remediation).toContain('celilo machine reclassify celilo-mgr --apply');
+  });
+
+  test('reports nothing for a machine whose labels agree', () => {
+    expect(auditMachineInterfaceZones({ machines: [{ hostname: 'pi-01', changes: [] }] })).toEqual(
+      [],
+    );
+  });
+});

package/src/services/audit/machine-interface-zones.ts

@@ -0,0 +1,43 @@
+/**
+ * Stored machine interface zone labels that no longer match the zone model.
+ *
+ * A label written at `machine add` is never revisited, so a zone-model change
+ * leaves every earlier machine describing a topology that no longer exists.
+ * Nothing said so until this check: celilo#1382 surfaced only as an upgrade
+ * failing with "no non-loopback interface in zone 'secure-mgmt'", on a machine
+ * that had one.
+ *
+ * `drift`, not `blocked`: a stale label breaks a deploy only when something
+ * resolves by zone, and the remediation is one command.
+ *
+ * Pure over a pre-computed diff — see services/machine-interface-zones.ts.
+ */
+
+import type { ZoneReclassification } from '../machine-interface-zones';
+import type { DriftFinding } from './types';
+
+export interface MachineInterfaceZonesAuditDeps {
+  /** One entry per machine, carrying only the labels that disagree. */
+  machines: Array<{ hostname: string; changes: ZoneReclassification[] }>;
+}
+
+export function auditMachineInterfaceZones(deps: MachineInterfaceZonesAuditDeps): DriftFinding[] {
+  return deps.machines
+    .filter((m) => m.changes.length > 0)
+    .map(({ hostname, changes }) => {
+      const named = changes
+        .map((c) => `${c.name} (${c.ipAddress}): ${c.from} → ${c.to}`)
+        .join(', ');
+      return {
+        category: 'interface_classification',
+        severity: 'drift',
+        code: 'stale_interface_zone_labels',
+        subject: hostname,
+        message: `${hostname}: ${changes.length} stored interface zone label(s) predate the current zone model — ${named}`,
+        details:
+          'Interface zones are decided once, at `machine add`, and never re-checked. When a zone is added or a subnet is declared for one that had none, every machine added earlier keeps the old label. Anything resolving a machine by zone then reports the interface as absent rather than as mislabelled.',
+        remediation: `celilo machine reclassify ${hostname} --apply`,
+        actionable: true,
+      };
+    });
+}

package/src/services/audit/reboot-pending.test.ts

@@ -0,0 +1,63 @@
+import { describe, expect, it } from 'bun:test';
+import { auditRebootPending } from './reboot-pending';
+
+const CLEAN = { pending: false, unmeasured: false, reasons: [] };
+const NOW = 1_700_000_000_000;
+
+describe('auditRebootPending', () => {
+  it('reports a pending reboot as drift, carrying the reason', () => {
+    const [finding] = auditRebootPending({
+      verdict: {
+        pending: true,
+        unmeasured: false,
+        reasons: ['the running kernel is 6.8.0-1063-raspi'],
+      },
+      marker: null,
+      now: () => NOW,
+    });
+
+    expect(finding.severity).toBe('drift');
+    expect(finding.code).toBe('reboot_pending');
+    expect(finding.message).toContain('6.8.0-1063-raspi');
+    expect(finding.remediation).toContain('celilo system reboot');
+  });
+
+  it('reports an unverified reboot as BLOCKED — a control plane may not have come back', () => {
+    const [finding] = auditRebootPending({
+      verdict: CLEAN,
+      marker: { bootIdBefore: 'a', requestedAt: NOW - 3 * 60 * 60 * 1000, reason: 'kernel' },
+      now: () => NOW,
+    });
+
+    expect(finding.severity).toBe('blocked');
+    expect(finding.code).toBe('reboot_unverified');
+    expect(finding.message).toContain('3h ago');
+  });
+
+  it('is silent on a host that is current and has no reboot outstanding', () => {
+    expect(auditRebootPending({ verdict: CLEAN, marker: null, now: () => NOW })).toEqual([]);
+  });
+
+  it('reports UNMEASURED rather than clean when the host could not be asked', () => {
+    const [finding] = auditRebootPending({
+      verdict: { pending: false, unmeasured: true, reasons: [] },
+      marker: null,
+      now: () => NOW,
+    });
+
+    expect(finding.severity).toBe('unmeasured');
+    expect(finding.actionable).toBe(false);
+  });
+
+  it('still reports the unverified reboot when the host itself is unmeasurable', () => {
+    // The two facts are independent: not being able to read the kernel does
+    // not make an outstanding reboot request go away.
+    const findings = auditRebootPending({
+      verdict: { pending: false, unmeasured: true, reasons: [] },
+      marker: { bootIdBefore: 'a', requestedAt: NOW, reason: '' },
+      now: () => NOW,
+    });
+
+    expect(findings.map((f) => f.code)).toEqual(['reboot_unverified', 'reboot_pending_unmeasured']);
+  });
+});

package/src/services/audit/reboot-pending.ts

@@ -0,0 +1,85 @@
+/**
+ * Pending reboot on the management server (celilo#1378).
+ *
+ * `apt-upgrade` printed needrestart's pending-kernel block and that was the
+ * only place the condition was ever visible — so it was visible exactly when
+ * an operator happened to run an upgrade, and invisible the rest of the time.
+ * celilo-mgr sat on a kernel it was not running for days that way.
+ *
+ * Two findings, and they are different facts:
+ *
+ *   reboot_pending    — the host needs a reboot and has not had one.
+ *   reboot_unverified — celilo ASKED for a reboot and nothing has confirmed
+ *                       the host came back. That is the more serious of the
+ *                       two: it is the shape of a control plane that went
+ *                       down and did not return, or returned without its
+ *                       dispatcher.
+ */
+
+import type { RebootMarker, RebootPendingVerdict } from '../system-reboot';
+import type { DriftFinding } from './types';
+
+export interface RebootPendingAuditDeps {
+  verdict: RebootPendingVerdict;
+  /** A reboot celilo requested that `verifyReturn` has not yet cleared. */
+  marker: RebootMarker | null;
+  /** Defaults to `Date.now()`. */
+  now?: () => number;
+}
+
+const HOUR = 60 * 60 * 1000;
+
+export function auditRebootPending(deps: RebootPendingAuditDeps): DriftFinding[] {
+  const findings: DriftFinding[] = [];
+  const now = (deps.now ?? Date.now)();
+
+  if (deps.marker !== null) {
+    const ageHours = Math.round((now - deps.marker.requestedAt) / HOUR);
+    findings.push({
+      category: 'reboot_pending',
+      severity: 'blocked',
+      code: 'reboot_unverified',
+      subject: 'system',
+      message: `celilo requested a reboot ${ageHours}h ago and nothing has confirmed the host came back.`,
+      details:
+        'The process that asks for a reboot goes down with the host, so the return is verified by a second call. An uncleared marker means that call never succeeded — either the host did not reboot, or it came back without its dispatcher.',
+      remediation:
+        'Run `celilo system reboot --verify`. It passes only once the boot id has changed and the dispatcher and CLI answer, and it names which of those is missing.',
+      actionable: true,
+    });
+  }
+
+  if (deps.verdict.unmeasured) {
+    findings.push({
+      category: 'reboot_pending',
+      severity: 'unmeasured',
+      code: 'reboot_pending_unmeasured',
+      subject: 'system',
+      message:
+        'Neither the reboot-required marker nor a kernel pair could be read, so nothing was compared.',
+      details:
+        'An absent /var/run/reboot-required is not evidence of a current kernel — a host without that path simply never writes one.',
+      remediation:
+        'Run `celilo system audit` on the management server itself (Debian/Ubuntu), where /var/run/reboot-required and `uname -r` are readable.',
+      actionable: false,
+    });
+    return findings;
+  }
+
+  if (deps.verdict.pending) {
+    findings.push({
+      category: 'reboot_pending',
+      severity: 'drift',
+      code: 'reboot_pending',
+      subject: 'system',
+      message: `The management server needs a reboot: ${deps.verdict.reasons.join('; ')}.`,
+      details:
+        'A kernel or service the host has installed is not the one it is running. Until it reboots, an upgrade that has been applied on disk has not taken effect.',
+      remediation:
+        'Reboot through celilo: `celilo system reboot --allow-reboot`. It refuses while a module operation is in flight, and `--verify` confirms the host came back.',
+      actionable: true,
+    });
+  }
+
+  return findings;
+}

package/src/services/audit/recurrence-gate.test.ts

@@ -38,7 +38,9 @@ import { auditInterfaceClassification } from './interface-classification';
 import { auditModuleIntegrity } from './module-integrity';
 import { auditModuleVersions } from './module-versions';
 import { auditPublicDns } from './public-dns';
+import { auditRebootPending } from './reboot-pending';
 import { auditSchema } from './schema';
+import { auditServerBunPin } from './server-bun-pin';
 import { auditTerraformPlan } from './terraform-plan';
 import { auditTrustedSources } from './trusted-sources';
 import type { DriftCategory, DriftFinding, DriftSeverity } from './types';
@@ -68,6 +70,25 @@ const GATE: Record<DriftCategory, GateEntry> = {
     because:
       'compares capability versions claimed in installed module manifests against the framework registry and deployed providers, all read from celilo own DB and local manifests — the subject is celilo itself and is always reachable',
   },
+  reboot_pending: {
+    probe: () =>
+      auditRebootPending({
+        verdict: { pending: false, unmeasured: true, reasons: [] },
+        marker: null,
+      }),
+    because:
+      'probed: a host with no /var/run/reboot-required and no readable kernel pair was not measured, and an absent marker is not a current kernel',
+  },
+  server_bun_pin: {
+    probe: () =>
+      auditServerBunPin({
+        pinnedVersion: null,
+        runningVersion: '1.4.2',
+        pinPath: '/etc/celilo/mise.pin.toml',
+      }),
+    because:
+      'probed: the deb pin file can be absent (a non-deb install), and then nothing was compared',
+  },
   browser_pin: {
     because:
       'compares a bundled client playwright revision against the locally provisioned browser binary, both on this machine — always reachable',