@celilo/cli 0.22.0 → 0.24.0

package/src/cli/commands/monitor.ts

@@ -10,12 +10,12 @@ import { defineEvents, openBus } from '@celilo/event-bus';
 import { getEventBusPath } from '../../config/paths';
 import { getDb } from '../../db/client';
 import type { MonitorKind } from '../../db/schema';
-import { parseIntervalMinutes } from '../../manifest/schema';
 import {
   isSchedulableBuiltin,
   runBuiltinCheckForMonitor,
 } from '../../services/alerting/builtin-source';
 import { loadModuleCoverage } from '../../services/alerting/coverage-source';
+import { loadModuleHealthCadences } from '../../services/alerting/health-cadence';
 import { HEALTH_COVERAGE_CHECK } from '../../services/alerting/health-coverage';
 import {
   createMonitor,
@@ -23,11 +23,18 @@ import {
   findMonitorByTarget,
   listMonitors,
   setMonitorEnabled,
+  updateMonitorInterval,
 } from '../../services/alerting/monitors';
 import { listPolicies } from '../../services/alerting/people';
 import { runOneMonitor } from '../../services/alerting/run-monitor';
 import { promoteReadyAlerts } from '../../services/alerting/store';
 import type { DriftCategory } from '../../services/audit/types';
+import {
+  MONITOR_INTERVAL_FLOOR_MINUTES,
+  cadenceSchema,
+  formatCadence,
+  parseCadence,
+} from '../../services/cadence';
 import { runModuleHealthCheck } from '../../services/health-runner';
 import type { CommandResult } from '../types';
 
@@ -65,18 +72,34 @@ function handleList(): CommandResult {
   const policyOf = (id: string | null) =>
     id ? (policies.get(id) ?? '(deleted policy)') : '— pages nobody';
 
+  // A `module_hook` row's stored interval is not what the sweep uses, so
+  // printing it would be a confident lie. Resolve the same way the sweep does.
+  const cadences = loadModuleHealthCadences(getDb());
+  const everyOf = (monitor: (typeof rows)[number]): string => {
+    if (monitor.kind !== 'module_hook') return `${monitor.intervalMinutes}m`;
+    const cadence = cadences.get(monitor.target)?.cadence ?? null;
+    return cadence === null ? '—' : formatCadence(cadence);
+  };
+
   const width = Math.max(6, ...rows.map((r) => r.target.length));
   const policyWidth = Math.max(6, ...rows.map((r) => policyOf(r.escalationPolicyId).length));
   console.log('');
   console.log(
-    `${'TARGET'.padEnd(width)}  ${'KIND'.padEnd(14)}  ${'EVERY'.padEnd(6)}  ${'POLICY'.padEnd(policyWidth)}  STATE`,
+    `${'TARGET'.padEnd(width)}  ${'KIND'.padEnd(14)}  ${'EVERY'.padEnd(8)}  ${'POLICY'.padEnd(policyWidth)}  STATE`,
   );
   for (const monitor of rows) {
-    const state = monitor.enabled ? 'enabled' : 'disabled';
+    const every = everyOf(monitor);
+    const state =
+      monitor.kind === 'module_hook'
+        ? every === 'manual' || every === '—'
+          ? 'not watched'
+          : 'watched'
+        : monitor.enabled
+          ? 'enabled'
+          : 'disabled';
     const suffix = monitor.lastRunAt ? '' : '  (never run)';
-    const every = `${monitor.intervalMinutes}m`;
     console.log(
-      `${monitor.target.padEnd(width)}  ${monitor.kind.padEnd(14)}  ${every.padEnd(6)}  ${policyOf(monitor.escalationPolicyId).padEnd(policyWidth)}  ${state}${suffix}`,
+      `${monitor.target.padEnd(width)}  ${monitor.kind.padEnd(14)}  ${every.padEnd(8)}  ${policyOf(monitor.escalationPolicyId).padEnd(policyWidth)}  ${state}${suffix}`,
     );
   }
   console.log('');
@@ -90,18 +113,6 @@ function handleAdd(args: string[], flags: Record<string, boolean | string>): Com
   }
 
   const db = getDb();
-  if (findMonitorByTarget(db, target)) {
-    return { success: false, error: `A monitor for "${target}" already exists.` };
-  }
-
-  const interval = typeof flags.interval === 'string' ? flags.interval : '15m';
-  const intervalMinutes = parseIntervalMinutes(interval);
-  if (intervalMinutes === null) {
-    return {
-      success: false,
-      error: `Invalid --interval "${interval}". Use a duration like "15m", "1h", or "1d".`,
-    };
-  }
 
   // A target naming an audit category is a built-in check; anything else is a
   // module's health_check hook. `isSchedulableBuiltin` is checked explicitly
@@ -113,6 +124,30 @@ function handleAdd(args: string[], flags: Record<string, boolean | string>): Com
       ? 'builtin_check'
       : 'module_hook';
 
+  // Checked BEFORE the already-exists check, so an operator reaching for the
+  // cadence knob is told where it moved rather than told the row exists. A
+  // module's cadence does not live on its row, so accepting `--interval` here
+  // would store a number nothing reads — refuse rather than ignore.
+  if (kind === 'module_hook' && typeof flags.interval === 'string') {
+    return { success: false, error: moduleCadenceRedirect(target, '--interval') };
+  }
+
+  if (findMonitorByTarget(db, target)) {
+    return {
+      success: false,
+      error:
+        kind === 'module_hook'
+          ? `A monitor for "${target}" already exists — a deploy creates one for every module with a health_check hook.\n\nTo change how often it runs:  celilo module config set ${target} health_check_interval <cadence>`
+          : `A monitor for "${target}" already exists.\n\nTo change how often it runs:  celilo monitor set-interval ${target} <cadence>`,
+    };
+  }
+
+  const interval = typeof flags.interval === 'string' ? flags.interval : '15m';
+  const invalid = validateMonitorCadence(interval);
+  if (invalid) return { success: false, error: invalid };
+  const cadence = parseCadence(interval);
+  const intervalMinutes = cadence !== null && cadence !== 'manual' ? cadence.minutes : 0;
+
   createMonitor(db, { kind, target, intervalMinutes });
 
   // Creating the first monitor is also what switches the sweep on. Registering
@@ -127,7 +162,10 @@ function handleAdd(args: string[], flags: Record<string, boolean | string>): Com
 
   return {
     success: true,
-    message: `Monitoring ${target} every ${interval} (sweep runs on timer.tick.5m)`,
+    message:
+      kind === 'module_hook'
+        ? `Watching ${target}. How often is per-module policy:  celilo module config set ${target} health_check_interval <cadence>`
+        : `Monitoring ${target} every ${interval} (sweep runs on timer.tick.5m)`,
   };
 }
 
@@ -167,11 +205,68 @@ function handleToggle(args: string[], enabled: boolean): CommandResult {
   const db = getDb();
   const monitor = findMonitorByTarget(db, target);
   if (!monitor) return { success: false, error: `No monitor for "${target}".` };
+  if (monitor.kind === 'module_hook') {
+    return { success: false, error: moduleCadenceRedirect(target, `monitor ${verb}`) };
+  }
 
   setMonitorEnabled(db, monitor.id, enabled, new Date());
   return { success: true, message: `Monitor for ${target} ${enabled ? 'enabled' : 'disabled'}` };
 }
 
+/**
+ * Two ways to change one module's cadence would disagree about what `module
+ * status` shows, so the fleet-level commands refuse a module target and name
+ * the per-module one. The targets are disjoint in practice — an operator has
+ * either a module or an audit check in hand — and an explicit error teaches
+ * better than silence.
+ */
+function moduleCadenceRedirect(target: string, what: string): string {
+  return (
+    `${what} does not apply to "${target}" — it is a module, and a module's cadence is per-module policy rather than a monitor setting. Its monitor row is created by the deploy.\n\n` +
+    `  celilo module config set ${target} health_check_interval 15m   # watch it every 15 minutes\n` +
+    `  celilo module config set ${target} health_check_interval manual # stop watching it\n` +
+    `  celilo module config unset ${target} health_check_interval      # follow the manifest again`
+  );
+}
+
+/** The floor comes from the alerting sweep's own tick, never a written-down number. */
+function validateMonitorCadence(value: string): string | null {
+  const result = cadenceSchema({ floorMinutes: MONITOR_INTERVAL_FLOOR_MINUTES }).safeParse(value);
+  if (result.success) return null;
+  return `Invalid interval "${value}".\n\n${result.error.issues.map((i) => i.message).join('\n')}`;
+}
+
+/**
+ * `celilo monitor set-interval <check> <cadence>` — re-cadence a built-in check
+ * without removing and recreating it, which would orphan its alert history.
+ */
+function handleSetInterval(args: string[]): CommandResult {
+  const [target, cadence] = args;
+  if (!target || !cadence) {
+    return { success: false, error: 'Usage: celilo monitor set-interval <check> <cadence>' };
+  }
+
+  const db = getDb();
+  const monitor = findMonitorByTarget(db, target);
+  if (!monitor) return { success: false, error: `No monitor for "${target}".` };
+  if (monitor.kind === 'module_hook') {
+    return { success: false, error: moduleCadenceRedirect(target, 'monitor set-interval') };
+  }
+
+  const invalid = validateMonitorCadence(cadence);
+  if (invalid) return { success: false, error: invalid };
+  const parsed = parseCadence(cadence);
+  if (parsed === null || parsed === 'manual') {
+    return {
+      success: false,
+      error: `A built-in check has no "manual" — disable it instead:  celilo monitor disable ${target}`,
+    };
+  }
+
+  updateMonitorInterval(db, monitor.id, parsed.minutes);
+  return { success: true, message: `${target} now runs every ${formatCadence(parsed)}` };
+}
+
 export async function handleMonitor(
   subcommand: string | undefined,
   args: string[],
@@ -189,10 +284,12 @@ export async function handleMonitor(
       return handleToggle(args, true);
     case 'disable':
       return handleToggle(args, false);
+    case 'set-interval':
+      return handleSetInterval(args);
     default:
       return {
         success: false,
-        error: `Unknown monitor subcommand: ${subcommand}\n\nUse: list, add, run, enable, disable`,
+        error: `Unknown monitor subcommand: ${subcommand}\n\nUse: list, add, run, set-interval, enable, disable`,
       };
   }
 }

package/src/cli/commands/proxmox-template-selection.ts

@@ -60,7 +60,7 @@ function compareDescending(a: ParsedUbuntu, b: ParsedUbuntu): number {
 
 /**
  * Build the option list used by the select prompt and tests.
- * Exported so tests can assert on filtering/sorting without mocking @clack.
+ * Exported so tests can assert on filtering/sorting without driving a prompt.
  */
 export function buildUbuntuOptions(appliances: ProxmoxAppliance[]): ParsedUbuntu[] {
   return appliances

package/src/cli/commands/status.ts

@@ -10,6 +10,7 @@ import { eq } from 'drizzle-orm';
 import { getDb } from '../../db/client';
 import { capabilities, moduleConfigs, modules, systemConfig, systemSecrets } from '../../db/schema';
 import { type ModuleManifest, getSingularSystemSpec } from '../../manifest/schema';
+import { formatPausedDuration } from '../../services/module-pause';
 import type { CommandResult } from '../types';
 
 /**
@@ -26,10 +27,21 @@ async function determineModuleStatus(
   generatedPath: string,
   dbState: string,
 ): Promise<{
-  status: 'IMPORTED' | 'CONFIGURED' | 'GENERATED' | 'DEPLOYED' | 'VERIFIED' | 'NEEDS_UPDATE';
+  status:
+    | 'IMPORTED'
+    | 'CONFIGURED'
+    | 'GENERATED'
+    | 'DEPLOYED'
+    | 'VERIFIED'
+    | 'NEEDS_UPDATE'
+    | 'PAUSED';
   missingCount?: number;
 }> {
-  // Deployed states come directly from the DB — don't infer from filesystem
+  // Deployed states come directly from the DB — don't infer from filesystem.
+  // PAUSED is checked FIRST: a paused module still has a generated/ directory
+  // and a full config, so every derivation below it would report it as an
+  // ordinary deployed module and the pause would be invisible here.
+  if (dbState === 'PAUSED') return { status: 'PAUSED' };
   if (dbState === 'VERIFIED') return { status: 'VERIFIED' };
   if (dbState === 'INSTALLED') return { status: 'DEPLOYED' };
 
@@ -120,6 +132,9 @@ export async function handleStatus(): Promise<CommandResult> {
         );
 
         // Status icon
+        // A paused module is deliberately out of service, so it must never
+        // carry the same ✓ as a healthy one — that is exactly how a pause turns
+        // into an outage nobody notices.
         const icon =
           statusInfo.status === 'VERIFIED' ||
           statusInfo.status === 'DEPLOYED' ||
@@ -128,7 +143,14 @@ export async function handleStatus(): Promise<CommandResult> {
             : '⚠';
 
         lines.push(`  ${icon} ${module.id} (v${module.version})`);
-        lines.push(`    Status: ${statusInfo.status}`);
+        if (statusInfo.status === 'PAUSED') {
+          // State plus AGE — a pause with alerting suppressed is only safe if
+          // how long it has run is impossible to miss (design D7).
+          lines.push(`    Status: PAUSED (${formatPausedDuration(module.pausedAt)})`);
+          if (module.pauseReason) lines.push(`    Paused: ${module.pauseReason}`);
+        } else {
+          lines.push(`    Status: ${statusInfo.status}`);
+        }
 
         // Type: VPS or Container
         const zone = getSingularSystemSpec(manifest)?.zone;

package/src/cli/commands/system-migrate.ts

@@ -14,6 +14,7 @@ import { createDbClient, findMigrationsFolder, getDb } from '../../db/client';
 import { runMigrationsOn } from '../../db/migrate';
 import { getMigrationStatus } from '../../db/migration-status';
 import { findSchemaDrift } from '../../db/schema-introspection';
+import { migrateMonitorCadences } from '../../services/alerting/cadence-migration';
 import { ensureBackupSweepSubscriber } from '../../services/backup-sweep';
 import { ensureOperationsSweepSubscriber } from '../../services/module-operations';
 import type { CommandResult } from '../types';
@@ -141,6 +142,13 @@ export async function handleSystemMigrate(
 
   ensureCoreSubscribers();
 
+  // A health-check cadence used to live on the monitor row and now resolves
+  // from `module_configs`. Without carrying the existing rows over, this very
+  // upgrade would silently revert every operator's cadence to the manifest's
+  // suggestion and resume watching modules they deliberately stopped watching.
+  // Idempotent: writes only where no override exists.
+  const cadences = migrateMonitorCadences(db);
+
   // Name the latest migration, not just a table count: "35 tables" reads the
   // same whether a column migration applied or silently did nothing (celilo#604).
   const status = getMigrationStatus(sqlite, findMigrationsFolder());
@@ -148,6 +156,12 @@ export async function handleSystemMigrate(
     applied > 0 ? `Applied ${applied} migration(s).` : 'Schema already up to date.',
     `Applied migrations: ${status.appliedCount} (latest: ${status.latestApplied ?? 'none'})`,
     `Schema current: ${drift.tableCount} tables, ${drift.columnCount} columns.`,
+    ...(cadences.written.size > 0
+      ? [
+          `Carried ${cadences.written.size} health-check cadence(s) into module config:`,
+          ...[...cadences.written].map(([moduleId, value]) => `  ${moduleId}: ${value}`),
+        ]
+      : []),
   ];
   return { success: true, message: lines.join('\n'), data: status };
 }

package/src/cli/commands/system-update.ts

@@ -31,8 +31,9 @@ import { fetchLatestCliVersion } from '../../services/audit/cli-version';
 import { unusedPublicDnsProbe } from '../../services/audit/public-dns';
 import { makeJournalReader, readAppliedMigrations } from '../../services/audit/schema';
 import { createModuleBackup, createSystemStateBackup } from '../../services/backup-create';
+import { BACKUP_SCHEDULE_CONFIG_KEY } from '../../services/backup-schedule';
 import { runAllHealthChecks, runModuleHealthCheck } from '../../services/health-runner';
-import { parseStoredConfigValue } from '../../services/module-config';
+import { configOverride, parseStoredConfigValue } from '../../services/module-config';
 import { deployModule } from '../../services/module-deploy';
 import { buildModuleGraph } from '../../services/update/dep-graph';
 import {
@@ -566,6 +567,7 @@ export async function handleSystemUpdate(
         id: m.id,
         state: m.state,
         manifest: m.manifestData as ModuleManifest,
+        scheduleOverride: configOverride(configsByModule.get(m.id), BACKUP_SCHEDULE_CONFIG_KEY),
         lastSuccessfulBackupAt: latestBackupByModule.get(m.id) ?? null,
       })),
     },
@@ -794,6 +796,7 @@ export function rebuildAuditDepsForRerun(
         id: m.id,
         state: m.state,
         manifest: m.manifestData as ModuleManifest,
+        scheduleOverride: configOverride(configsByModule.get(m.id), BACKUP_SCHEDULE_CONFIG_KEY),
         lastSuccessfulBackupAt: priorBackupByModule.get(m.id) ?? null,
       })),
     },

package/src/cli/completion.ts

@@ -13,6 +13,7 @@ import { listBackups } from '../services/backup-metadata';
 import { listBackupStorages } from '../services/backup-storage';
 import { listContainerServices } from '../services/container-service';
 import { listMachines } from '../services/machine-pool';
+import { FRAMEWORK_CONFIG_KEYS } from './commands/module-config';
 
 /**
  * Get completion suggestions based on current command context
@@ -41,6 +42,7 @@ export async function getCompletions(words: string[], current: number): Promise<
       'alerts',
       'escalation-policy',
       'events',
+      'firewall',
       'help',
       'hook',
       'ipam',
@@ -194,13 +196,15 @@ export async function getCompletions(words: string[], current: number): Promise<
       'terraform-unlock',
       'types',
       'validate',
+      'pause',
+      'unpause',
     ];
     return filterSuggestions(subcommands, args[1] || '');
   }
 
-  // Module config subcommands (celilo module config set/get)
+  // Module config subcommands (celilo module config set/get/unset)
   if (command === 'module' && args[1] === 'config' && currentIndex === 2) {
-    const subcommands = ['set', 'get'];
+    const subcommands = ['set', 'get', 'unset'];
     return filterSuggestions(subcommands, args[2] || '');
   }
 
@@ -216,11 +220,11 @@ export async function getCompletions(words: string[], current: number): Promise<
     return filterSuggestions(subcommands, args[2] || '');
   }
 
-  // Module config set/get - complete with module IDs
+  // Module config set/get/unset - complete with module IDs
   if (
     command === 'module' &&
     args[1] === 'config' &&
-    (args[2] === 'set' || args[2] === 'get') &&
+    (args[2] === 'set' || args[2] === 'get' || args[2] === 'unset') &&
     currentIndex === 3
   ) {
     const db = getDb();
@@ -229,11 +233,16 @@ export async function getCompletions(words: string[], current: number): Promise<
     return filterSuggestions(moduleIds, args[3] || '');
   }
 
-  // Module config set/get <module-id> - complete with config variable names
+  // Module config set/get/unset <module-id> - complete with config key names.
+  // Framework keys (backup cadence, upgrade policy…) are offered on EVERY
+  // module: they describe how celilo treats a module, so a module never
+  // declares them and completion built from the manifest alone could not see
+  // them — which is why `auto_upgrade` and `upgrade_policy` were uncompletable
+  // for as long as they have existed.
   if (
     command === 'module' &&
     args[1] === 'config' &&
-    (args[2] === 'set' || args[2] === 'get') &&
+    (args[2] === 'set' || args[2] === 'get' || args[2] === 'unset') &&
     currentIndex === 4
   ) {
     const db = getDb();
@@ -242,13 +251,15 @@ export async function getCompletions(words: string[], current: number): Promise<
       .from(modules)
       .where(eq(modules.id, args[3] || ''))
       .get();
+    const frameworkKeys = Object.keys(FRAMEWORK_CONFIG_KEYS);
     if (module?.manifestData) {
       const manifest = module.manifestData as ModuleManifest;
       const varNames = (manifest.variables?.owns || [])
         .filter((v) => v.source === 'user' || !v.source)
         .map((v) => v.name);
-      return filterSuggestions(varNames, args[4] || '');
+      return filterSuggestions([...varNames, ...frameworkKeys], args[4] || '');
     }
+    return filterSuggestions(frameworkKeys, args[4] || '');
   }
 
   // Module secret subcommands (celilo module secret set/list)
@@ -393,6 +404,16 @@ export async function getCompletions(words: string[], current: number): Promise<
   }
 
   // Machine subcommands
+  if (command === 'firewall' && currentIndex === 1) {
+    // `interface` is the only group. `acknowledge` and `enforce` were deleted in
+    // the design amendments — there is no policy toggle to complete.
+    return filterSuggestions(['interface'], args[1] || '');
+  }
+
+  if (command === 'firewall' && args[1] === 'interface' && currentIndex === 2) {
+    return filterSuggestions(['list'], args[2] || '');
+  }
+
   if (command === 'machine' && currentIndex === 1) {
     const subcommands = ['add', 'list', 'status', 'remove', 'earmark', 'detect'];
     return filterSuggestions(subcommands, args[1] || '');
@@ -437,6 +458,8 @@ export async function getCompletions(words: string[], current: number): Promise<
       'terraform-unlock',
       'verify',
       'audit', // deprecation alias for `verify`
+      'pause',
+      'unpause',
     ];
     if (moduleCommands.includes(args[1] || '')) {
       const db = getDb();
@@ -480,7 +503,10 @@ export async function getCompletions(words: string[], current: number): Promise<
 
   // Monitor subcommands
   if (command === 'monitor' && currentIndex === 1) {
-    return filterSuggestions(['list', 'add', 'run', 'enable', 'disable'], args[1] || '');
+    return filterSuggestions(
+      ['list', 'add', 'run', 'set-interval', 'enable', 'disable'],
+      args[1] || '',
+    );
   }
 
   // Monitor targets - a module ID or one of celilo's own schedulable checks.
@@ -488,7 +514,11 @@ export async function getCompletions(words: string[], current: number): Promise<
   // a new built-in check is completable the moment it is schedulable.
   if (
     command === 'monitor' &&
-    (args[1] === 'add' || args[1] === 'run' || args[1] === 'enable' || args[1] === 'disable') &&
+    (args[1] === 'add' ||
+      args[1] === 'run' ||
+      args[1] === 'set-interval' ||
+      args[1] === 'enable' ||
+      args[1] === 'disable') &&
     currentIndex === 2
   ) {
     const db = getDb();

package/src/cli/fuel-gauge.ts

@@ -10,7 +10,7 @@
  */
 
 import { stdout } from 'node:process';
-import * as p from '@clack/prompts';
+import { log } from '@celilo/cli-display';
 import { getActiveDisplay } from './prompts';
 
 /**
@@ -173,7 +173,7 @@ export class FuelGauge {
     if (!this.running) return;
 
     this.cleanup();
-    p.log.info(`${this.title} (backgrounded)`);
+    log.info(`${this.title} (backgrounded)`);
 
     if (this.onBackground) {
       this.onBackground();
@@ -208,9 +208,9 @@ export class FuelGauge {
     this.cleanup();
 
     if (success) {
-      p.log.success(this.title);
+      log.success(this.title);
     } else {
-      p.log.error(this.title);
+      log.error(this.title);
       console.log('');
       console.log(colors.dim('Last output:'));
       const errorLines = this.outputLines.slice(-this.errorDisplayLines);