@celilo/cli 0.22.0 → 0.24.0

package/src/services/module-config.ts

@@ -151,6 +151,39 @@ export function parseStoredConfigValue(
   }
 }
 
+/**
+ * A module's whole operator config as a plain key → typed-value map — the shape
+ * every policy resolver takes.
+ */
+export function loadModuleConfigs(db: DbClient, moduleId: string): Record<string, unknown> {
+  const configs: Record<string, unknown> = {};
+  for (const row of db
+    .select()
+    .from(moduleConfigs)
+    .where(eq(moduleConfigs.moduleId, moduleId))
+    .all()) {
+    configs[row.key] = parseStoredConfigValue(row);
+  }
+  return configs;
+}
+
+/**
+ * The raw string form of an operator override from an already-loaded config
+ * map, or undefined when the key is unset.
+ *
+ * Framework policy keys (cadences, retention, upgrade controls) are stored
+ * through the same typed path as any other config, so a cadence arrives as a
+ * string and a retention count as a number. Every resolver takes the string
+ * form and parses it itself, so this is the one place that flattening happens.
+ */
+export function configOverride(
+  configs: Record<string, unknown> | undefined,
+  key: string,
+): string | undefined {
+  const raw = configs?.[key];
+  return raw === undefined || raw === null ? undefined : String(raw);
+}
+
 /**
  * Get module configuration value
  * Returns parsed value (primitive or complex type)

package/src/services/module-deploy.ts

@@ -350,7 +350,7 @@ async function deployModuleImpl(
 
   // Terminal-responder: when running on a TTY, this subscribes to
   // `config.required.*` / `secret.required.*` / `ensure.required.*`
-  // events and prompts via clack. Other responder shapes (Claude
+  // events and prompts on the terminal. Other responder shapes (Claude
   // subagent, `celilo events respond` from another shell, autoresponder
   // daemon) compete on the bus; first reply wins. See
   // infra/openspec/changes/interactive-deploys-via-event-bus/proposal.md.

package/src/services/module-pause-observability.test.ts

@@ -0,0 +1,224 @@
+/**
+ * The parts of pause that exist so a pause cannot become a silent outage
+ * (design D7), plus the alerting suppressor.
+ *
+ * Every check here is asserted in BOTH directions per Rule 7.6 — a gate nobody
+ * has seen fail is not a gate, and "no paused module was reported" is exactly
+ * the output a broken detector produces.
+ */
+
+import { describe, expect, test } from 'bun:test';
+import { mkdtempSync } from 'node:fs';
+import { tmpdir } from 'node:os';
+import { join } from 'node:path';
+import { type DbClient, createDbClient } from '../db/client';
+import { type ModuleState, modules } from '../db/schema';
+import { moduleAlertKey } from './alerting/keys';
+import { findSuppressor } from './alerting/suppression';
+import { checkPausedModules } from './fleet-checks';
+import {
+  describeMachineStopInfra,
+  describePausedModule,
+  listPausedModules,
+  pausedAmong,
+} from './module-pause';
+
+function makeDb(): DbClient {
+  const dir = mkdtempSync(join(tmpdir(), 'celilo-pause-obs-'));
+  process.env.CELILO_DB_PATH = join(dir, 'celilo.db');
+  return createDbClient({ path: process.env.CELILO_DB_PATH });
+}
+
+function insert(
+  db: DbClient,
+  id: string,
+  state: ModuleState,
+  opts: { pausedAt?: Date; reason?: string } = {},
+): void {
+  db.insert(modules)
+    .values({
+      id,
+      name: id,
+      version: '1.0.0',
+      state,
+      manifestData: { id, name: id, version: '1.0.0' },
+      sourcePath: `/tmp/${id}`,
+      pausedAt: state === 'PAUSED' ? (opts.pausedAt ?? new Date()) : null,
+      pauseReason: state === 'PAUSED' ? (opts.reason ?? null) : null,
+    })
+    .run();
+}
+
+describe('system doctor reports ANY paused module (task 6.2 / 6.4)', () => {
+  test('with nothing paused the check passes', () => {
+    const db = makeDb();
+    insert(db, 'caddy', 'VERIFIED');
+
+    const finding = checkPausedModules(db);
+    expect(finding.status).toBe('ok');
+    expect(finding.summary).toBe('nothing paused');
+  });
+
+  test('PROVE IT FAILS: one paused module is a doctor FAILURE naming it and its age', () => {
+    const db = makeDb();
+    insert(db, 'caddy', 'VERIFIED');
+    insert(db, 'greenwave', 'PAUSED', {
+      pausedAt: new Date(Date.now() - 3 * 24 * 60 * 60 * 1000),
+      reason: 'edge router swap',
+    });
+
+    const finding = checkPausedModules(db);
+
+    expect(finding.status).toBe('fail');
+    expect(finding.summary).toContain('greenwave');
+    // The DURATION is what separates a maintenance window from a forgotten one.
+    expect(finding.detail.join('\n')).toContain('3d');
+    expect(finding.detail.join('\n')).toContain('edge router swap');
+    expect(finding.remediation).toContain('celilo module unpause');
+  });
+
+  test('there is no threshold — a pause taken seconds ago already fails', () => {
+    // Decided at review: a pause is a degraded state, full stop. A threshold is
+    // just something to tune until the detector stops firing.
+    const db = makeDb();
+    insert(db, 'greenwave', 'PAUSED', { pausedAt: new Date() });
+    expect(checkPausedModules(db).status).toBe('fail');
+  });
+});
+
+describe('listPausedModules / pausedAmong', () => {
+  test('reads only PAUSED rows, and reads back the reason', () => {
+    const db = makeDb();
+    insert(db, 'caddy', 'INSTALLED');
+    insert(db, 'greenwave', 'PAUSED', { reason: 'edge router swap' });
+    insert(db, 'authentik', 'ERROR');
+
+    const paused = listPausedModules(db);
+    expect(paused.map((m) => m.id)).toEqual(['greenwave']);
+    expect(paused[0].pauseReason).toBe('edge router swap');
+  });
+
+  test('an ERROR module is NOT paused — the two states are distinct', () => {
+    // Worth pinning: ERROR is *pausable*, which is easy to misread as "errored
+    // counts as paused" and would make the doctor check fire on every failure.
+    const db = makeDb();
+    insert(db, 'authentik', 'ERROR');
+    expect(listPausedModules(db)).toEqual([]);
+  });
+
+  test('pausedAmong filters a candidate set in one query', () => {
+    const db = makeDb();
+    insert(db, 'caddy', 'PAUSED');
+    insert(db, 'authentik', 'INSTALLED');
+    expect(pausedAmong(db, ['caddy', 'authentik', 'absent'])).toEqual(new Set(['caddy']));
+  });
+
+  test('pausedAmong on an empty list does not query at all', () => {
+    const db = makeDb();
+    expect(pausedAmong(db, [])).toEqual(new Set());
+  });
+});
+
+describe('the management-API warning is emitted, and NOT emitted (task 6.5 / 6.6)', () => {
+  // `fleetWarnings()` in api/serve.ts is the production caller; this pins the
+  // query + rendering it depends on. The inverse assertion is the one that
+  // stops the warning becoming noise nobody reads.
+  test('with nothing paused there is nothing to warn about', () => {
+    const db = makeDb();
+    insert(db, 'caddy', 'VERIFIED');
+    expect(listPausedModules(db)).toHaveLength(0);
+  });
+
+  test('with something paused the warning names the module AND its age', () => {
+    const db = makeDb();
+    insert(db, 'greenwave', 'PAUSED', {
+      pausedAt: new Date(Date.now() - 5 * 60 * 60 * 1000),
+      reason: 'edge router swap',
+    });
+
+    const rendered = listPausedModules(db).map((m) => describePausedModule(m));
+    expect(rendered).toEqual(['greenwave (5h, "edge router swap")']);
+  });
+
+  test('a pause with no reason still renders its age', () => {
+    const db = makeDb();
+    insert(db, 'greenwave', 'PAUSED', { pausedAt: new Date(Date.now() - 60 * 60 * 1000) });
+    expect(describePausedModule(listPausedModules(db)[0])).toBe('greenwave (1h)');
+  });
+});
+
+describe('a paused module is suppressed BY THE PAUSE, not anonymously (task 2.3)', () => {
+  const topology = { moduleSystems: [], zoneProviders: [] };
+
+  test('PROVE IT FIRES: an unpaused module with nothing firing is NOT suppressed', () => {
+    expect(
+      findSuppressor({
+        key: moduleAlertKey('caddy'),
+        firingKeys: new Set(),
+        suppressible: true,
+        modulesInDeployWindow: new Set(),
+        pausedModules: new Set(),
+        topology,
+      }),
+    ).toBeNull();
+  });
+
+  test('the same alert IS suppressed once the module is paused, attributed to the pause', () => {
+    expect(
+      findSuppressor({
+        key: moduleAlertKey('caddy'),
+        firingKeys: new Set(),
+        suppressible: true,
+        modulesInDeployWindow: new Set(),
+        pausedModules: new Set(['caddy']),
+        topology,
+      }),
+    ).toEqual({ kind: 'paused', moduleId: 'caddy' });
+  });
+
+  test('a pause does not suppress OTHER modules', () => {
+    expect(
+      findSuppressor({
+        key: moduleAlertKey('authentik'),
+        firingKeys: new Set(),
+        suppressible: true,
+        modulesInDeployWindow: new Set(),
+        pausedModules: new Set(['caddy']),
+        topology,
+      }),
+    ).toBeNull();
+  });
+
+  test('a self-monitor is never suppressed, even by a pause', () => {
+    // A cascading failure must not silence the component reporting the cascade.
+    expect(
+      findSuppressor({
+        key: moduleAlertKey('caddy'),
+        firingKeys: new Set(),
+        suppressible: false,
+        modulesInDeployWindow: new Set(),
+        pausedModules: new Set(['caddy']),
+        topology,
+      }),
+    ).toBeNull();
+  });
+});
+
+describe('--stop-infra acts only on celilo-provisioned infrastructure (design D2, revised)', () => {
+  // The RULING is about ownership, not capability, and the distinction lives in
+  // the message the operator reads — so the message is what gets pinned.
+  //
+  // celilo could SSH into a machine and stop something. It declines to, because
+  // a machine-pool system is operator-pre-provisioned, may predate celilo, and
+  // may run work celilo was never told about. Reporting "celilo cannot identify
+  // the service unit" would imply a capability gap where the truth is that this
+  // is not celilo's to stop.
+  test('the machine-hosted report reads as not-applicable, not as a missing feature', () => {
+    const report = describeMachineStopInfra('iot', 'homebridge');
+
+    expect(report).toContain('not applicable');
+    expect(report).toContain('celilo provisioned');
+    // Must NOT frame it as something celilo would do if only it could.
+    expect(report).not.toMatch(/cannot determine|does not know which|unit/i);
+  });
+});

package/src/services/module-pause-quiescence.test.ts

@@ -0,0 +1,163 @@
+/**
+ * Control-plane quiescence (tasks 2.1, 2.2, 2.5).
+ *
+ * Every assertion is made in BOTH directions per Rule 7.6. "The hook did not
+ * fire" is the same observation you get from a test that was never wired up, so
+ * each case first proves the UNPAUSED module gets through the guard.
+ */
+
+import { afterEach, beforeEach, describe, expect, test } from 'bun:test';
+import { mkdtempSync } from 'node:fs';
+import { tmpdir } from 'node:os';
+import { join } from 'node:path';
+import { eq } from 'drizzle-orm';
+import { handleEventsRunHook } from '../cli/commands/events';
+import { type DbClient, createDbClient } from '../db/client';
+import { type ModuleState, modules } from '../db/schema';
+import { runNamedHook } from '../hooks/run-named-hook';
+import type { HookLogger } from '../hooks/types';
+
+const SILENT_LOGGER: HookLogger = {
+  info: () => {},
+  warn: () => {},
+  error: () => {},
+  debug: () => {},
+} as unknown as HookLogger;
+
+let db: DbClient;
+
+/**
+ * A manifest declaring the timer- and event-driven hooks that are the runtime
+ * hazard — the ones the dispatcher fires against celilo-mgr's module store and
+ * that can call into a provider that is gone.
+ */
+function manifestWithHooks(id: string) {
+  return {
+    id,
+    name: id,
+    version: '1.0.0',
+    celilo_contract: '1.0',
+    provides: { capabilities: [] },
+    requires: { capabilities: [] },
+    hooks: {
+      on_install: { script: 'scripts/on-install.ts' },
+      on_uninstall: { script: 'scripts/on-uninstall.ts' },
+      on_system_event: { script: 'scripts/on-system-event.ts' },
+      refresh_registrations: { script: 'scripts/refresh-registrations.ts' },
+    },
+    subscriptions: [{ name: 'refresh', pattern: 'timer.tick.15m', hook: 'refresh_registrations' }],
+  };
+}
+
+function insertModule(id: string, state: ModuleState): void {
+  db.insert(modules)
+    .values({
+      id,
+      name: id,
+      version: '1.0.0',
+      state,
+      manifestData: manifestWithHooks(id),
+      sourcePath: join(tmpdir(), 'celilo-nonexistent-module'),
+      pausedAt: state === 'PAUSED' ? new Date() : null,
+      pauseReason: state === 'PAUSED' ? 'edge router swap' : null,
+    })
+    .run();
+}
+
+beforeEach(() => {
+  const dir = mkdtempSync(join(tmpdir(), 'celilo-quiesce-'));
+  process.env.CELILO_DB_PATH = join(dir, 'celilo.db');
+  process.env.CELILO_EVENT_BUS_PATH = join(dir, 'events.db');
+  db = createDbClient({ path: process.env.CELILO_DB_PATH });
+});
+
+afterEach(() => {
+  db.$client.close();
+});
+
+describe('runNamedHook refuses to run dispatched work on a paused module', () => {
+  test('PROVE IT RUNS: an unpaused module is NOT short-circuited by the pause guard', async () => {
+    insertModule('caddy', 'INSTALLED');
+    const result = await runNamedHook('caddy', 'on_system_event', db, SILENT_LOGGER);
+
+    // The hook script does not exist, so this fails downstream — the point is
+    // only that it got PAST the guard, which `skippedPaused` proves it did.
+    expect(result.skippedPaused).toBeUndefined();
+  });
+
+  test('a paused module is skipped instead', async () => {
+    insertModule('caddy', 'PAUSED');
+    const result = await runNamedHook('caddy', 'on_system_event', db, SILENT_LOGGER);
+
+    expect(result.skippedPaused).toBe(true);
+    // Success, not failure: the module is deliberately quiesced. A failure here
+    // would be retried by the bus and then alerted on, paging the operator
+    // about the pause they took themselves.
+    expect(result.success).toBe(true);
+  });
+
+  test('a paused module is skipped for timer-driven hooks too', async () => {
+    insertModule('technitium', 'PAUSED');
+    const result = await runNamedHook('technitium', 'refresh_registrations', db, SILENT_LOGGER);
+    expect(result.skippedPaused).toBe(true);
+  });
+
+  test.each([['on_install'], ['on_uninstall']] as const)(
+    '%s is EXEMPT — it must run while the module is still paused',
+    async (hookName) => {
+      // on_install is how unpause redeploys the module back to life, and
+      // on_uninstall is how a paused provider is removed — which is the entire
+      // point of pausing its consumers. Blocking either would deadlock the
+      // whole design.
+      insertModule('greenwave', 'PAUSED');
+      const result = await runNamedHook(
+        'greenwave',
+        hookName as 'on_install' | 'on_uninstall',
+        db,
+        SILENT_LOGGER,
+      );
+      expect(result.skippedPaused).toBeUndefined();
+    },
+  );
+});
+
+describe('the bus dispatch entry point skips a paused module (task 2.1)', () => {
+  // `celilo events run-hook <module> <sub>` is what every `hook:` subscription
+  // resolves to, so this is where an event delivery lands.
+  test('PROVE IT PROCEEDS: an unpaused module gets past the guard to subscription lookup', async () => {
+    insertModule('caddy', 'INSTALLED');
+    const result = await handleEventsRunHook(['caddy', 'no-such-subscription', '1']);
+
+    // Reaching the "no such subscription" error proves the pause guard let it
+    // through; a paused module never gets this far.
+    expect(result.success).toBe(false);
+    expect(result.success === false ? result.error : '').toContain('no subscription named');
+  });
+
+  test('a paused module is skipped before any subscription is resolved', async () => {
+    insertModule('caddy', 'PAUSED');
+    const result = await handleEventsRunHook(['caddy', 'no-such-subscription', '1']);
+
+    expect(result.success).toBe(true);
+    expect(result.success === true ? result.message : '').toContain('paused');
+  });
+});
+
+describe('resyncAllSubscriptions cannot re-arm a paused module', () => {
+  test('a paused module is outside the deployed set the resync rebuilds from', () => {
+    // The resync selects INSTALLED/VERIFIED. That allow-list is what makes
+    // quiescence survive a restore (which starts events.db empty) — worth
+    // pinning, because widening it to "not IMPORTED" would silently un-pause
+    // every paused module on the next resync.
+    insertModule('caddy', 'PAUSED');
+    insertModule('authentik', 'INSTALLED');
+
+    const deployed = db
+      .select({ id: modules.id })
+      .from(modules)
+      .where(eq(modules.state, 'PAUSED'))
+      .all();
+
+    expect(deployed.map((m) => m.id)).toEqual(['caddy']);
+  });
+});