@celilo/cli 0.21.0 → 0.23.0

package/src/services/fleet-checks.test.ts

@@ -42,6 +42,19 @@ function seedHeartbeat(
   );
 }
 
+/** Abandon `count` deliveries to one subscriber, oldest first. */
+function seedFailed(bus: Bus, count: number): void {
+  const sub = bus.subscribe({ name: 'namecheap.ddns', pattern: 'ddns.refresh', handler: 'echo' });
+  for (let i = 0; i < count; i++) {
+    const event = bus.emitRaw('ddns.refresh', { n: i });
+    bus.markFailed(
+      { eventId: event.id, subscriberId: sub.id },
+      new Error('handler timed out after 30000ms'),
+      { abandoned: true },
+    );
+  }
+}
+
 /** Write a supervisor unit file so readInstalledUnit(scope) sees it. */
 function installFakeUnit(home: string, scope: 'user' | 'system' = 'user', systemRoot = '/'): void {
   const path = getDaemonUnitPath('linux', home, scope, systemRoot);
@@ -199,6 +212,39 @@ describe('checkDispatcher', () => {
     expect(f.status).toBe('warn');
     expect(f.detail.join(' ')).toContain('not emitting on schedule');
   });
+
+  // celilo#623 — the old check read `failedDeliveries({ limit: 50 }).length`,
+  // so on celilo-mgr it printed a literal `50` that meant "at least 50" and
+  // read as an exact count. 137 > any limit anyone would pick.
+  it('reports the TRUE total of failed deliveries, not the read limit', () => {
+    seedHeartbeat(bus, { startedAt: now - MINUTE, lastHeartbeat: now - 1000 });
+    installFakeUnit(home);
+    seedFailed(bus, 137);
+
+    const f = checkDispatcher(bus, { now: now, home, platform: 'linux' });
+    expect(f.status).toBe('warn');
+    expect(f.detail.join(' ')).toContain('137 failed/abandoned delivery(ies) total');
+    expect(f.detail.join(' ')).not.toContain('50 failed');
+  });
+
+  // The stored error is double-wrapped for every row written before the
+  // serializeError fix; those live 90 days, so doctor must unwrap them.
+  it('renders the sample error text, not {"message":"[object Object]"}', () => {
+    seedHeartbeat(bus, { startedAt: now - MINUTE, lastHeartbeat: now - 1000 });
+    installFakeUnit(home);
+    seedFailed(bus, 1);
+    bus.db.run('UPDATE deliveries SET last_error = ?', [
+      JSON.stringify({
+        message: '[object Object]',
+        value: { message: 'handler exited with code 1' },
+      }),
+    ]);
+
+    const f = checkDispatcher(bus, { now: now, home, platform: 'linux' });
+    const detail = f.detail.join(' ');
+    expect(detail).toContain('handler exited with code 1');
+    expect(detail).not.toContain('[object Object]');
+  });
 });
 
 describe('checkSubscribers + checkCapabilityProviders', () => {

package/src/services/fleet-checks.ts

@@ -17,7 +17,7 @@
  * rendering + `--fix` orchestration lives in the doctor command.
  */
 
-import type { Bus } from '@celilo/event-bus';
+import { type Bus, describeError } from '@celilo/event-bus';
 import { inArray } from 'drizzle-orm';
 import { getModuleStoragePath } from '../config/paths';
 import { type DbClient, findMigrationsFolder } from '../db/client';
@@ -38,6 +38,7 @@ import {
   readInstalledUnit,
   unitMainPid,
 } from './events-daemon';
+import { describePausedModule, listPausedModules } from './module-pause';
 import { resolveSubscription } from './module-subscriptions';
 
 /**
@@ -343,12 +344,21 @@ export function checkDispatcher(bus: Bus, opts: DispatcherCheckOptions = {}): Fl
     }
   }
 
-  const failed = bus.failedDeliveries({ limit: 50 });
-  if (failed.length > 0) {
+  // A TRUE total, not `failedDeliveries().length` — that saturates at its LIMIT
+  // and printed a literal `50` on celilo-mgr that read as a count (celilo#623).
+  // The newest row dates the backlog: a big total whose newest entry is days old
+  // is drained history, not active bleeding.
+  const { total: failedTotal } = bus.failedDeliveryTotals();
+  if (failedTotal > 0) {
     statuses.push('warn');
-    const sample = failed[0]?.lastError ? ` (e.g. ${failed[0].lastError.split('\n')[0]})` : '';
-    detail.push(`${failed.length} failed/abandoned delivery(ies)${sample}`);
-    remediations.push('inspect failed deliveries and re-emit/repair as needed');
+    const newest = bus.failedDeliveries({ limit: 1 })[0];
+    const age = newest?.finishedAt
+      ? `, most recent ${Math.round((now - newest.finishedAt) / 60000)}min ago`
+      : '';
+    detail.push(`${failedTotal} failed/abandoned delivery(ies) total${age}`);
+    const sample = describeError(newest?.lastError ?? null)?.split('\n')[0];
+    if (sample) detail.push(`  example (newest, not the only one): ${sample}`);
+    remediations.push('`celilo events list-failed` to see them; re-emit/repair as needed');
   }
 
   const status = worst(statuses);
@@ -845,6 +855,52 @@ export function checkControlPlaneNetwork(db: DbClient): FleetFinding {
   };
 }
 
+/**
+ * Any paused module is a doctor failure — no threshold, whatever its age
+ * (openspec/changes/module-pause-lifecycle, design D7, closed at review).
+ *
+ * A pause deliberately switches OFF the alerting that would otherwise report
+ * the module as down, so the paused-ness itself has to be the signal. A
+ * duration threshold was considered and dropped: there is no number of hours
+ * after which a deliberate outage becomes acceptable, and a configurable one is
+ * just an invitation to tune the detector until it stops firing.
+ *
+ * The fleet had just run 20 hours of failing forgejo backups whose only symptom
+ * was a column of `0 B` rows that read as healthy hourly cadence. Same failure
+ * shape; this is the check that would have named it.
+ */
+export function checkPausedModules(db: DbClient): FleetFinding {
+  const paused = listPausedModules(db);
+
+  if (paused.length === 0) {
+    return {
+      id: 'paused-modules',
+      title: 'No module is paused (a pause suppresses its own alerting)',
+      status: 'ok',
+      summary: 'nothing paused',
+      detail: [],
+      remediation: null,
+      autoFixable: false,
+    };
+  }
+
+  const names = paused.map((m) => describePausedModule(m));
+  return {
+    id: 'paused-modules',
+    title: 'No module is paused (a pause suppresses its own alerting)',
+    status: 'fail',
+    summary: `${paused.length} module(s) paused: ${paused.map((m) => m.id).join(', ')}`,
+    detail: [
+      ...names.map((n) => `paused: ${n}`),
+      'a paused module receives no dispatched work, runs no health checks, and has its alerts suppressed',
+      'it is still deployed and may still be serving traffic — pause does not stop the data plane',
+    ],
+    remediation:
+      'bring each back with "celilo module unpause <id>" (which redeploys it, rebinding its capabilities), or remove it if the pause was permanent',
+    autoFixable: false,
+  };
+}
+
 export async function runFleetChecks(
   bus: Bus,
   db: DbClient,
@@ -856,6 +912,7 @@ export async function runFleetChecks(
     checkSubscribers(bus, db),
     checkCapabilityProviders(db),
     checkControlPlaneNetwork(db),
+    checkPausedModules(db),
     await checkServiceDns(db),
   ];
 }

package/src/services/module-deploy.ts

@@ -350,7 +350,7 @@ async function deployModuleImpl(
 
   // Terminal-responder: when running on a TTY, this subscribes to
   // `config.required.*` / `secret.required.*` / `ensure.required.*`
-  // events and prompts via clack. Other responder shapes (Claude
+  // events and prompts on the terminal. Other responder shapes (Claude
   // subagent, `celilo events respond` from another shell, autoresponder
   // daemon) compete on the bus; first reply wins. See
   // infra/openspec/changes/interactive-deploys-via-event-bus/proposal.md.

package/src/services/module-pause-observability.test.ts

@@ -0,0 +1,224 @@
+/**
+ * The parts of pause that exist so a pause cannot become a silent outage
+ * (design D7), plus the alerting suppressor.
+ *
+ * Every check here is asserted in BOTH directions per Rule 7.6 — a gate nobody
+ * has seen fail is not a gate, and "no paused module was reported" is exactly
+ * the output a broken detector produces.
+ */
+
+import { describe, expect, test } from 'bun:test';
+import { mkdtempSync } from 'node:fs';
+import { tmpdir } from 'node:os';
+import { join } from 'node:path';
+import { type DbClient, createDbClient } from '../db/client';
+import { type ModuleState, modules } from '../db/schema';
+import { moduleAlertKey } from './alerting/keys';
+import { findSuppressor } from './alerting/suppression';
+import { checkPausedModules } from './fleet-checks';
+import {
+  describeMachineStopInfra,
+  describePausedModule,
+  listPausedModules,
+  pausedAmong,
+} from './module-pause';
+
+function makeDb(): DbClient {
+  const dir = mkdtempSync(join(tmpdir(), 'celilo-pause-obs-'));
+  process.env.CELILO_DB_PATH = join(dir, 'celilo.db');
+  return createDbClient({ path: process.env.CELILO_DB_PATH });
+}
+
+function insert(
+  db: DbClient,
+  id: string,
+  state: ModuleState,
+  opts: { pausedAt?: Date; reason?: string } = {},
+): void {
+  db.insert(modules)
+    .values({
+      id,
+      name: id,
+      version: '1.0.0',
+      state,
+      manifestData: { id, name: id, version: '1.0.0' },
+      sourcePath: `/tmp/${id}`,
+      pausedAt: state === 'PAUSED' ? (opts.pausedAt ?? new Date()) : null,
+      pauseReason: state === 'PAUSED' ? (opts.reason ?? null) : null,
+    })
+    .run();
+}
+
+describe('system doctor reports ANY paused module (task 6.2 / 6.4)', () => {
+  test('with nothing paused the check passes', () => {
+    const db = makeDb();
+    insert(db, 'caddy', 'VERIFIED');
+
+    const finding = checkPausedModules(db);
+    expect(finding.status).toBe('ok');
+    expect(finding.summary).toBe('nothing paused');
+  });
+
+  test('PROVE IT FAILS: one paused module is a doctor FAILURE naming it and its age', () => {
+    const db = makeDb();
+    insert(db, 'caddy', 'VERIFIED');
+    insert(db, 'greenwave', 'PAUSED', {
+      pausedAt: new Date(Date.now() - 3 * 24 * 60 * 60 * 1000),
+      reason: 'edge router swap',
+    });
+
+    const finding = checkPausedModules(db);
+
+    expect(finding.status).toBe('fail');
+    expect(finding.summary).toContain('greenwave');
+    // The DURATION is what separates a maintenance window from a forgotten one.
+    expect(finding.detail.join('\n')).toContain('3d');
+    expect(finding.detail.join('\n')).toContain('edge router swap');
+    expect(finding.remediation).toContain('celilo module unpause');
+  });
+
+  test('there is no threshold — a pause taken seconds ago already fails', () => {
+    // Decided at review: a pause is a degraded state, full stop. A threshold is
+    // just something to tune until the detector stops firing.
+    const db = makeDb();
+    insert(db, 'greenwave', 'PAUSED', { pausedAt: new Date() });
+    expect(checkPausedModules(db).status).toBe('fail');
+  });
+});
+
+describe('listPausedModules / pausedAmong', () => {
+  test('reads only PAUSED rows, and reads back the reason', () => {
+    const db = makeDb();
+    insert(db, 'caddy', 'INSTALLED');
+    insert(db, 'greenwave', 'PAUSED', { reason: 'edge router swap' });
+    insert(db, 'authentik', 'ERROR');
+
+    const paused = listPausedModules(db);
+    expect(paused.map((m) => m.id)).toEqual(['greenwave']);
+    expect(paused[0].pauseReason).toBe('edge router swap');
+  });
+
+  test('an ERROR module is NOT paused — the two states are distinct', () => {
+    // Worth pinning: ERROR is *pausable*, which is easy to misread as "errored
+    // counts as paused" and would make the doctor check fire on every failure.
+    const db = makeDb();
+    insert(db, 'authentik', 'ERROR');
+    expect(listPausedModules(db)).toEqual([]);
+  });
+
+  test('pausedAmong filters a candidate set in one query', () => {
+    const db = makeDb();
+    insert(db, 'caddy', 'PAUSED');
+    insert(db, 'authentik', 'INSTALLED');
+    expect(pausedAmong(db, ['caddy', 'authentik', 'absent'])).toEqual(new Set(['caddy']));
+  });
+
+  test('pausedAmong on an empty list does not query at all', () => {
+    const db = makeDb();
+    expect(pausedAmong(db, [])).toEqual(new Set());
+  });
+});
+
+describe('the management-API warning is emitted, and NOT emitted (task 6.5 / 6.6)', () => {
+  // `fleetWarnings()` in api/serve.ts is the production caller; this pins the
+  // query + rendering it depends on. The inverse assertion is the one that
+  // stops the warning becoming noise nobody reads.
+  test('with nothing paused there is nothing to warn about', () => {
+    const db = makeDb();
+    insert(db, 'caddy', 'VERIFIED');
+    expect(listPausedModules(db)).toHaveLength(0);
+  });
+
+  test('with something paused the warning names the module AND its age', () => {
+    const db = makeDb();
+    insert(db, 'greenwave', 'PAUSED', {
+      pausedAt: new Date(Date.now() - 5 * 60 * 60 * 1000),
+      reason: 'edge router swap',
+    });
+
+    const rendered = listPausedModules(db).map((m) => describePausedModule(m));
+    expect(rendered).toEqual(['greenwave (5h, "edge router swap")']);
+  });
+
+  test('a pause with no reason still renders its age', () => {
+    const db = makeDb();
+    insert(db, 'greenwave', 'PAUSED', { pausedAt: new Date(Date.now() - 60 * 60 * 1000) });
+    expect(describePausedModule(listPausedModules(db)[0])).toBe('greenwave (1h)');
+  });
+});
+
+describe('a paused module is suppressed BY THE PAUSE, not anonymously (task 2.3)', () => {
+  const topology = { moduleSystems: [], zoneProviders: [] };
+
+  test('PROVE IT FIRES: an unpaused module with nothing firing is NOT suppressed', () => {
+    expect(
+      findSuppressor({
+        key: moduleAlertKey('caddy'),
+        firingKeys: new Set(),
+        suppressible: true,
+        modulesInDeployWindow: new Set(),
+        pausedModules: new Set(),
+        topology,
+      }),
+    ).toBeNull();
+  });
+
+  test('the same alert IS suppressed once the module is paused, attributed to the pause', () => {
+    expect(
+      findSuppressor({
+        key: moduleAlertKey('caddy'),
+        firingKeys: new Set(),
+        suppressible: true,
+        modulesInDeployWindow: new Set(),
+        pausedModules: new Set(['caddy']),
+        topology,
+      }),
+    ).toEqual({ kind: 'paused', moduleId: 'caddy' });
+  });
+
+  test('a pause does not suppress OTHER modules', () => {
+    expect(
+      findSuppressor({
+        key: moduleAlertKey('authentik'),
+        firingKeys: new Set(),
+        suppressible: true,
+        modulesInDeployWindow: new Set(),
+        pausedModules: new Set(['caddy']),
+        topology,
+      }),
+    ).toBeNull();
+  });
+
+  test('a self-monitor is never suppressed, even by a pause', () => {
+    // A cascading failure must not silence the component reporting the cascade.
+    expect(
+      findSuppressor({
+        key: moduleAlertKey('caddy'),
+        firingKeys: new Set(),
+        suppressible: false,
+        modulesInDeployWindow: new Set(),
+        pausedModules: new Set(['caddy']),
+        topology,
+      }),
+    ).toBeNull();
+  });
+});
+
+describe('--stop-infra acts only on celilo-provisioned infrastructure (design D2, revised)', () => {
+  // The RULING is about ownership, not capability, and the distinction lives in
+  // the message the operator reads — so the message is what gets pinned.
+  //
+  // celilo could SSH into a machine and stop something. It declines to, because
+  // a machine-pool system is operator-pre-provisioned, may predate celilo, and
+  // may run work celilo was never told about. Reporting "celilo cannot identify
+  // the service unit" would imply a capability gap where the truth is that this
+  // is not celilo's to stop.
+  test('the machine-hosted report reads as not-applicable, not as a missing feature', () => {
+    const report = describeMachineStopInfra('iot', 'homebridge');
+
+    expect(report).toContain('not applicable');
+    expect(report).toContain('celilo provisioned');
+    // Must NOT frame it as something celilo would do if only it could.
+    expect(report).not.toMatch(/cannot determine|does not know which|unit/i);
+  });
+});

package/src/services/module-pause-quiescence.test.ts

@@ -0,0 +1,163 @@
+/**
+ * Control-plane quiescence (tasks 2.1, 2.2, 2.5).
+ *
+ * Every assertion is made in BOTH directions per Rule 7.6. "The hook did not
+ * fire" is the same observation you get from a test that was never wired up, so
+ * each case first proves the UNPAUSED module gets through the guard.
+ */
+
+import { afterEach, beforeEach, describe, expect, test } from 'bun:test';
+import { mkdtempSync } from 'node:fs';
+import { tmpdir } from 'node:os';
+import { join } from 'node:path';
+import { eq } from 'drizzle-orm';
+import { handleEventsRunHook } from '../cli/commands/events';
+import { type DbClient, createDbClient } from '../db/client';
+import { type ModuleState, modules } from '../db/schema';
+import { runNamedHook } from '../hooks/run-named-hook';
+import type { HookLogger } from '../hooks/types';
+
+const SILENT_LOGGER: HookLogger = {
+  info: () => {},
+  warn: () => {},
+  error: () => {},
+  debug: () => {},
+} as unknown as HookLogger;
+
+let db: DbClient;
+
+/**
+ * A manifest declaring the timer- and event-driven hooks that are the runtime
+ * hazard — the ones the dispatcher fires against celilo-mgr's module store and
+ * that can call into a provider that is gone.
+ */
+function manifestWithHooks(id: string) {
+  return {
+    id,
+    name: id,
+    version: '1.0.0',
+    celilo_contract: '1.0',
+    provides: { capabilities: [] },
+    requires: { capabilities: [] },
+    hooks: {
+      on_install: { script: 'scripts/on-install.ts' },
+      on_uninstall: { script: 'scripts/on-uninstall.ts' },
+      on_system_event: { script: 'scripts/on-system-event.ts' },
+      refresh_registrations: { script: 'scripts/refresh-registrations.ts' },
+    },
+    subscriptions: [{ name: 'refresh', pattern: 'timer.tick.15m', hook: 'refresh_registrations' }],
+  };
+}
+
+function insertModule(id: string, state: ModuleState): void {
+  db.insert(modules)
+    .values({
+      id,
+      name: id,
+      version: '1.0.0',
+      state,
+      manifestData: manifestWithHooks(id),
+      sourcePath: join(tmpdir(), 'celilo-nonexistent-module'),
+      pausedAt: state === 'PAUSED' ? new Date() : null,
+      pauseReason: state === 'PAUSED' ? 'edge router swap' : null,
+    })
+    .run();
+}
+
+beforeEach(() => {
+  const dir = mkdtempSync(join(tmpdir(), 'celilo-quiesce-'));
+  process.env.CELILO_DB_PATH = join(dir, 'celilo.db');
+  process.env.CELILO_EVENT_BUS_PATH = join(dir, 'events.db');
+  db = createDbClient({ path: process.env.CELILO_DB_PATH });
+});
+
+afterEach(() => {
+  db.$client.close();
+});
+
+describe('runNamedHook refuses to run dispatched work on a paused module', () => {
+  test('PROVE IT RUNS: an unpaused module is NOT short-circuited by the pause guard', async () => {
+    insertModule('caddy', 'INSTALLED');
+    const result = await runNamedHook('caddy', 'on_system_event', db, SILENT_LOGGER);
+
+    // The hook script does not exist, so this fails downstream — the point is
+    // only that it got PAST the guard, which `skippedPaused` proves it did.
+    expect(result.skippedPaused).toBeUndefined();
+  });
+
+  test('a paused module is skipped instead', async () => {
+    insertModule('caddy', 'PAUSED');
+    const result = await runNamedHook('caddy', 'on_system_event', db, SILENT_LOGGER);
+
+    expect(result.skippedPaused).toBe(true);
+    // Success, not failure: the module is deliberately quiesced. A failure here
+    // would be retried by the bus and then alerted on, paging the operator
+    // about the pause they took themselves.
+    expect(result.success).toBe(true);
+  });
+
+  test('a paused module is skipped for timer-driven hooks too', async () => {
+    insertModule('technitium', 'PAUSED');
+    const result = await runNamedHook('technitium', 'refresh_registrations', db, SILENT_LOGGER);
+    expect(result.skippedPaused).toBe(true);
+  });
+
+  test.each([['on_install'], ['on_uninstall']] as const)(
+    '%s is EXEMPT — it must run while the module is still paused',
+    async (hookName) => {
+      // on_install is how unpause redeploys the module back to life, and
+      // on_uninstall is how a paused provider is removed — which is the entire
+      // point of pausing its consumers. Blocking either would deadlock the
+      // whole design.
+      insertModule('greenwave', 'PAUSED');
+      const result = await runNamedHook(
+        'greenwave',
+        hookName as 'on_install' | 'on_uninstall',
+        db,
+        SILENT_LOGGER,
+      );
+      expect(result.skippedPaused).toBeUndefined();
+    },
+  );
+});
+
+describe('the bus dispatch entry point skips a paused module (task 2.1)', () => {
+  // `celilo events run-hook <module> <sub>` is what every `hook:` subscription
+  // resolves to, so this is where an event delivery lands.
+  test('PROVE IT PROCEEDS: an unpaused module gets past the guard to subscription lookup', async () => {
+    insertModule('caddy', 'INSTALLED');
+    const result = await handleEventsRunHook(['caddy', 'no-such-subscription', '1']);
+
+    // Reaching the "no such subscription" error proves the pause guard let it
+    // through; a paused module never gets this far.
+    expect(result.success).toBe(false);
+    expect(result.success === false ? result.error : '').toContain('no subscription named');
+  });
+
+  test('a paused module is skipped before any subscription is resolved', async () => {
+    insertModule('caddy', 'PAUSED');
+    const result = await handleEventsRunHook(['caddy', 'no-such-subscription', '1']);
+
+    expect(result.success).toBe(true);
+    expect(result.success === true ? result.message : '').toContain('paused');
+  });
+});
+
+describe('resyncAllSubscriptions cannot re-arm a paused module', () => {
+  test('a paused module is outside the deployed set the resync rebuilds from', () => {
+    // The resync selects INSTALLED/VERIFIED. That allow-list is what makes
+    // quiescence survive a restore (which starts events.db empty) — worth
+    // pinning, because widening it to "not IMPORTED" would silently un-pause
+    // every paused module on the next resync.
+    insertModule('caddy', 'PAUSED');
+    insertModule('authentik', 'INSTALLED');
+
+    const deployed = db
+      .select({ id: modules.id })
+      .from(modules)
+      .where(eq(modules.state, 'PAUSED'))
+      .all();
+
+    expect(deployed.map((m) => m.id)).toEqual(['caddy']);
+  });
+});