@celilo/cli 0.22.0 → 0.23.0

package/src/cli/commands/module-pause.ts

@@ -0,0 +1,317 @@
+/**
+ * `celilo module pause <id>` / `celilo module unpause <id>`.
+ *
+ * A thin adapter (Rule 10.5): parse flags, build the plan, confirm, execute,
+ * render. All the decisions live in `services/module-pause.ts` (pure planning)
+ * and the injected deps below (the side effects).
+ *
+ * See openspec/changes/module-pause-lifecycle/.
+ */
+
+import { join } from 'node:path';
+import { eq } from 'drizzle-orm';
+import { ProxmoxClient } from '../../api-clients/proxmox';
+import { getModuleStoragePath } from '../../config/paths';
+import { type DbClient, getDb } from '../../db/client';
+import { moduleSystems, modules } from '../../db/schema';
+import type { ModuleManifest } from '../../manifest/schema';
+import { ModuleManifestSchema } from '../../manifest/schema';
+import { askConfirm, withInterviewSession } from '../../services/bus-interview';
+import { getServiceCredentials } from '../../services/container-service';
+import { deployModule } from '../../services/module-deploy';
+import { checkInFlight } from '../../services/module-operations';
+import { completeOperation, failOperation, startOperation } from '../../services/module-operations';
+import {
+  type ExecutionReport,
+  type InfraStopOutcome,
+  type ModuleSnapshot,
+  type PauseDeps,
+  type PausePlan,
+  PauseRefusedError,
+  actedOn,
+  describeMachineStopInfra,
+  executePause,
+  executeUnpause,
+  planPause,
+  planUnpause,
+} from '../../services/module-pause';
+import {
+  registerModuleSubscriptions,
+  unregisterModuleSubscriptions,
+} from '../../services/module-subscriptions';
+import { getArg, hasFlag } from '../parser';
+import { log } from '../prompts';
+import type { CommandResult } from '../types';
+
+/**
+ * Every module celilo knows about, in the shape the planner wants. The planner
+ * is pure, so the whole fleet is read once here rather than queried per step.
+ *
+ * A module whose stored manifest no longer parses is dropped rather than
+ * failing the command: it cannot be a graph node, and refusing to pause the
+ * fleet because one unrelated manifest went stale would be the wrong trade.
+ */
+function loadFleet(db: DbClient): ModuleSnapshot[] {
+  const snapshots: ModuleSnapshot[] = [];
+  for (const row of db.select().from(modules).all()) {
+    const parsed = ModuleManifestSchema.safeParse(row.manifestData);
+    if (!parsed.success) continue;
+    snapshots.push({
+      id: row.id,
+      state: row.state,
+      pausedAt: row.pausedAt,
+      pauseReason: row.pauseReason,
+      manifest: parsed.data,
+    });
+  }
+  return snapshots;
+}
+
+/** moduleId → description, for the in-flight refusal (task 3.8). */
+function inFlightByModule(): Map<string, string> {
+  const map = new Map<string, string>();
+  for (const conflict of checkInFlight()) {
+    map.set(conflict.operation.moduleId, conflict.describe);
+  }
+  return map;
+}
+
+/**
+ * `--stop-infra` (design D2, revised). Opt-in, and deliberately NOT what a pause
+ * means: pausing `caddy` to swap the *firewall* must not take every website
+ * down.
+ *
+ * It acts ONLY on infrastructure celilo provisioned for this module:
+ *   - celilo-provisioned LXC/VM -> stopped; celilo created it, so it is celilo's
+ *   - machine-pool system       -> not applicable, by design (see below)
+ *   - systemless driver         -> nothing to stop
+ *
+ * The flag is a convenience for "I actually want the box off". Pause's real job
+ * is control-plane quiescence, and the provider swap this feature exists for
+ * never needs the flag at all.
+ */
+async function stopModuleInfrastructure(db: DbClient, moduleId: string): Promise<InfraStopOutcome> {
+  const systems = db.select().from(moduleSystems).where(eq(moduleSystems.moduleId, moduleId)).all();
+
+  if (systems.length === 0) {
+    // A driver module (`greenwave`, `axon`, `namecheap`) declares no
+    // `requires.system` — it talks to a device over HTTP. Nothing to stop, and
+    // that is a normal outcome to report, not an error (spec scenario
+    // "Shutdown on a systemless driver module reports nothing to stop").
+    return { stopped: false, detail: 'no infrastructure to stop' };
+  }
+
+  const detail: string[] = [];
+  for (const system of systems) {
+    if (system.infraType === 'machine') {
+      // NOT APPLICABLE, by design — not a missing feature (design D2, revised).
+      //
+      // `--stop-infra` acts only on infrastructure celilo PROVISIONED for the
+      // module. A machine-pool system is operator-pre-provisioned: it may
+      // predate celilo, and it may run things celilo has never heard of — not
+      // merely other celilo modules, but arbitrary operator work. Powering it
+      // off, or stopping services on it, reaches outside what celilo owns.
+      //
+      // Same principle as the sizing rule: a module must not own a host-level
+      // fact, because the host outlives any one module's config. Framing this
+      // as "celilo cannot identify the service unit" would be wrong — it
+      // implies a capability gap, when the answer is that this is not celilo's
+      // to stop.
+      detail.push(describeMachineStopInfra(system.hostname, moduleId));
+      continue;
+    }
+
+    if (system.vmid == null || !system.serviceId) {
+      detail.push(`${system.hostname}: no container recorded, nothing to stop`);
+      continue;
+    }
+
+    const credentials = await getServiceCredentials(system.serviceId);
+    if (!('api_url' in credentials)) {
+      detail.push(`${system.hostname}: container service is not Proxmox, nothing to stop`);
+      continue;
+    }
+
+    const client = new ProxmoxClient(credentials);
+    const result = await client.setGuestPower(system.vmid, 'lxc', 'shutdown');
+    detail.push(
+      result.success
+        ? `stopped ${system.hostname} (vmid ${system.vmid})`
+        : `could not stop ${system.hostname} (vmid ${system.vmid}): ${result.message}`,
+    );
+  }
+
+  return { stopped: true, detail: detail.join('; ') };
+}
+
+function buildDeps(db: DbClient): PauseDeps {
+  return {
+    db,
+    unsubscribe: (moduleId) => {
+      // The primary quiescence mechanism: with no subscriber rows the
+      // dispatcher has nothing to deliver to. `run-named-hook` guards the
+      // paths that do not go through the bus.
+      unregisterModuleSubscriptions(moduleId);
+    },
+    resubscribe: (moduleId) => {
+      // A deploy does NOT re-register subscriptions (only import and
+      // `module update` do), so unpause has to — otherwise the module comes
+      // back deployed but permanently deaf.
+      const row = db.select().from(modules).where(eq(modules.id, moduleId)).get();
+      if (!row) return;
+      const manifest = row.manifestData as ModuleManifest;
+      registerModuleSubscriptions(manifest, join(getModuleStoragePath(), moduleId));
+    },
+    redeploy: async (moduleId) => {
+      try {
+        const result = await deployModule(moduleId, db, {});
+        return { success: result.success, error: result.error };
+      } catch (err) {
+        return { success: false, error: err instanceof Error ? err.message : String(err) };
+      }
+    },
+    stopInfrastructure: (moduleId) => stopModuleInfrastructure(db, moduleId),
+    startOperation,
+    completeOperation,
+    failOperation,
+    now: () => new Date(),
+    log: (message) => log.info(message),
+  };
+}
+
+/** The ordered plan, rendered for `--dry-run` and for the confirmation. */
+function renderPlan(plan: PausePlan): string {
+  const verb = plan.action === 'pause' ? 'Pause' : 'Unpause';
+  const order = plan.action === 'pause' ? 'consumers first' : 'providers first';
+  const lines = [
+    plan.cascade
+      ? `${verb} ${plan.requested} and its transitive consumers (${order}):`
+      : `${verb} ${plan.requested}:`,
+  ];
+  plan.steps.forEach((step, index) => {
+    const suffix = step.disposition === 'act' ? '' : ` — skip (${step.note})`;
+    lines.push(`  ${index + 1}. ${step.moduleId}${suffix}`);
+  });
+  if (plan.stopInfra) {
+    lines.push('', 'Infrastructure will also be stopped (--stop-infra).');
+  }
+  return lines.join('\n');
+}
+
+function renderReport(report: ExecutionReport): string {
+  const lines = report.outcomes.map((o) => {
+    const mark = o.result === 'acted' ? '✓' : o.result === 'skipped' ? '·' : '✗';
+    return `  ${mark} ${o.moduleId}${o.detail ? ` — ${o.detail}` : ''}`;
+  });
+  const acted = report.outcomes.filter((o) => o.result === 'acted').length;
+  const skipped = report.outcomes.filter((o) => o.result === 'skipped').length;
+  const failed = report.outcomes.filter((o) => o.result === 'failed').length;
+  lines.push('', `${acted} changed, ${skipped} already done, ${failed} failed.`);
+  return lines.join('\n');
+}
+
+/**
+ * Confirmation for a cascade. An event-bus interview question, never a stdin
+ * prompt (design D6), so the operation is drivable headlessly — by CI, by the
+ * MCP, by a remote responder. `--yes` satisfies it without asking.
+ */
+async function confirmCascade(plan: PausePlan, yes: boolean): Promise<boolean> {
+  if (!plan.cascade || yes) return true;
+  const affected = actedOn(plan);
+  if (affected.length === 0) return true;
+
+  return withInterviewSession(() =>
+    askConfirm({
+      scope: `module-${plan.action}:${plan.requested}`,
+      key: 'cascade',
+      message: `${plan.action === 'pause' ? 'Pause' : 'Unpause'} ${affected.length} module(s): ${affected.join(', ')}?`,
+      description: renderPlan(plan),
+      defaultValue: false,
+    }),
+  );
+}
+
+export async function handleModulePause(
+  args: string[],
+  flags: Record<string, string | boolean> = {},
+): Promise<CommandResult> {
+  const moduleId = getArg(args, 0);
+  if (!moduleId) {
+    return {
+      success: false,
+      error:
+        'Module ID is required\n\nUsage: celilo module pause <id> [--cascade] [--stop-infra] [--dry-run] [--yes] [--reason "..."]',
+    };
+  }
+
+  const db = getDb();
+  const reason = typeof flags.reason === 'string' ? flags.reason : null;
+
+  let plan: PausePlan;
+  try {
+    plan = planPause({
+      moduleId,
+      fleet: loadFleet(db),
+      cascade: hasFlag(flags, 'cascade'),
+      stopInfra: hasFlag(flags, 'stop-infra'),
+      inFlight: inFlightByModule(),
+    });
+  } catch (err) {
+    if (err instanceof PauseRefusedError) return { success: false, error: err.message };
+    throw err;
+  }
+
+  if (hasFlag(flags, 'dry-run')) {
+    return { success: true, message: `${renderPlan(plan)}\n\n(dry run — nothing was changed)` };
+  }
+
+  if (!(await confirmCascade(plan, hasFlag(flags, 'yes')))) {
+    return { success: false, error: 'Cancelled — nothing was paused' };
+  }
+
+  const report = await executePause(plan, buildDeps(db), reason);
+  return report.success
+    ? { success: true, message: renderReport(report) }
+    : { success: false, error: renderReport(report) };
+}
+
+export async function handleModuleUnpause(
+  args: string[],
+  flags: Record<string, string | boolean> = {},
+): Promise<CommandResult> {
+  const moduleId = getArg(args, 0);
+  if (!moduleId) {
+    return {
+      success: false,
+      error:
+        'Module ID is required\n\nUsage: celilo module unpause <id> [--cascade] [--dry-run] [--yes]',
+    };
+  }
+
+  const db = getDb();
+
+  let plan: PausePlan;
+  try {
+    plan = planUnpause({
+      moduleId,
+      fleet: loadFleet(db),
+      cascade: hasFlag(flags, 'cascade'),
+    });
+  } catch (err) {
+    if (err instanceof PauseRefusedError) return { success: false, error: err.message };
+    throw err;
+  }
+
+  if (hasFlag(flags, 'dry-run')) {
+    return { success: true, message: `${renderPlan(plan)}\n\n(dry run — nothing was changed)` };
+  }
+
+  if (!(await confirmCascade(plan, hasFlag(flags, 'yes')))) {
+    return { success: false, error: 'Cancelled — nothing was unpaused' };
+  }
+
+  const report = await executeUnpause(plan, buildDeps(db));
+  return report.success
+    ? { success: true, message: renderReport(report) }
+    : { success: false, error: renderReport(report) };
+}

package/src/cli/commands/module-remove.ts

@@ -24,6 +24,11 @@ import {
 } from '../../services/celilo-events';
 import { getContainerService, getServiceCredentials } from '../../services/container-service';
 import { completeOperation, failOperation, startOperation } from '../../services/module-operations';
+import {
+  type DependentCandidate,
+  describeRemovalRefusal,
+  findRemovalBlockers,
+} from '../../services/remove-guard';
 import { cleanupWebRoutesForModule } from '../../services/web-route-cleanup';
 import { getArg, hasFlag, validateRequiredArgs } from '../parser';
 import { log } from '../prompts';
@@ -81,23 +86,26 @@ export async function handleModuleRemove(
     .all();
 
   if (providedCapabilities.length > 0) {
-    const providedNames = new Set(providedCapabilities.map((c) => c.capabilityName));
+    const providedNames = providedCapabilities.map((c) => c.capabilityName);
     const otherModules = db.select().from(modules).where(ne(modules.id, moduleId)).all();
 
-    const dependents = otherModules
-      .filter((m) => {
-        const parsed = ModuleManifestSchema.safeParse(m.manifestData);
-        if (!parsed.success) return false;
-        return (parsed.data.requires?.capabilities ?? []).some((req) =>
-          providedNames.has(req.name),
-        );
-      })
-      .map((m) => m.id);
-
-    if (dependents.length > 0) {
+    const candidates: DependentCandidate[] = [];
+    for (const m of otherModules) {
+      const parsed = ModuleManifestSchema.safeParse(m.manifestData);
+      if (!parsed.success) continue;
+      candidates.push({
+        id: m.id,
+        manifest: parsed.data,
+        paused: m.state === 'PAUSED',
+        deployed: !(['IMPORTED', 'VALIDATED', 'CONFIGURED'] as string[]).includes(m.state),
+      });
+    }
+
+    const blockers = findRemovalBlockers(providedNames, candidates);
+    if (blockers.length > 0) {
       return {
         success: false,
-        error: `Cannot remove '${moduleId}': the following installed modules depend on its capabilities: ${dependents.join(', ')}`,
+        error: describeRemovalRefusal(moduleId, blockers),
       };
     }
   }
@@ -163,9 +171,9 @@ async function performModuleRemove(
   // prompt on hook failure.
   const manifestForHook = module.manifestData as { hooks?: { on_uninstall?: unknown } } | undefined;
   if (manifestForHook?.hooks?.on_uninstall) {
-    // Match build-stream's TTY detection: in non-TTY contexts (tests,
-    // pipes, CI) skipAnimation prevents the gauge's setInterval and
-    // raw-stdin handlers from blocking process exit.
+    // Match build-stream's TTY detection: in non-TTY contexts (tests, pipes,
+    // CI) skipAnimation prevents the gauge's setInterval and raw-stdin
+    // handlers from blocking process exit.
     const isInteractive = process.stdout.isTTY && process.stdin.isTTY;
     const gauge = new FuelGauge(`${moduleId}: on_uninstall`, {
       skipAnimation: !isInteractive,
@@ -173,35 +181,65 @@ async function performModuleRemove(
     gauge.start();
     const logger = createGaugeLogger(gauge, moduleId, 'on_uninstall');
     const hookResult = await runNamedHook(moduleId, 'on_uninstall', db, logger, {});
+
     if (hookResult.success) {
       gauge.stop(true);
     } else {
       gauge.stop(false);
-      log.warn(`on_uninstall failed: ${hookResult.error ?? 'unknown error'}`);
-      if (force) {
-        log.info('--force set, continuing removal despite hook failure');
-      } else if (!isInteractive) {
-        // Non-TTY (tests, scripts, piped input). Don't try to prompt —
-        // @clack/prompts hangs on stdin in that context. Default
-        // behaviour: continue with removal so a hook crash doesn't
-        // wedge automation. Operators who want strict failure can
-        // re-run `celilo module run-hook <id> on_uninstall` manually.
-        log.warn('Non-interactive context detected; continuing removal despite hook failure');
-      } else {
-        const proceed = await withInterviewSession(() =>
-          askConfirm({
-            scope: `module-remove:${moduleId}`,
-            key: 'continue_after_hook_failure',
-            message:
-              'on_uninstall hook failed. Continue removing the module anyway? ' +
-              '(some external state may not be cleaned up)',
-            defaultValue: false,
-          }),
-        );
-        if (!proceed) {
-          return { success: false, error: 'Removal cancelled after on_uninstall failure' };
-        }
+      const reason = hookResult.error ?? 'unknown error';
+
+      if (!force) {
+        // STOP, and leave the module ERRORED. Do not ask.
+        //
+        // `on_uninstall` is what withdraws a module's cross-module state —
+        // caddy unexposes its ports and clears its Caddyfile, a firewall
+        // provider withdraws its forwards. If it failed, that state is still
+        // out there and we do not know how much of it the hook managed before
+        // dying. Deleting the module now orphans whatever is left, with no
+        // record of what to go clean up.
+        //
+        // This previously asked "continue removing anyway?" in a TTY and
+        // silently continued when not one. Both were wrong. The prompt offers
+        // the orphaning decision at the least informed possible moment — inside
+        // a Y/N, before anyone has looked at the machine — and the non-TTY
+        // default made orphaning the norm for every scripted removal.
+        //
+        // The module is marked ERROR rather than left reading INSTALLED,
+        // because the previous behaviour recorded the failure only in a
+        // `module_operations` row: `module list` and `system doctor` both
+        // showed the module as perfectly healthy while its teardown had failed.
+        //
+        // The way forward is deliberate: go look, clean up by hand, then
+        // re-run with --force, which means "I have remediated; drop the
+        // record."
+        db.update(modules)
+          .set({
+            state: 'ERROR',
+            errorMessage: `on_uninstall failed: ${reason}`,
+            updatedAt: new Date(),
+          })
+          .where(eq(modules.id, moduleId))
+          .run();
+
+        return {
+          success: false,
+          error: [
+            `Cannot remove '${moduleId}': its on_uninstall hook failed, so cross-module state it registered (port forwards, DNS records, web routes) may still exist.`,
+            '',
+            `  ${reason}`,
+            '',
+            `'${moduleId}' is now marked ERROR and has NOT been removed. Inspect what the hook left behind, clean it up, then re-run:`,
+            `  celilo module remove ${moduleId} --force`,
+            '',
+            '--force means "I have remediated this by hand; delete the record anyway".',
+          ].join('\n'),
+        };
       }
+
+      log.warn(`on_uninstall failed: ${reason}`);
+      log.info(
+        '--force set: continuing removal. Any state the hook left behind is yours to clean up.',
+      );
     }
   }
 

package/src/cli/commands/module-status.ts

@@ -62,10 +62,9 @@ export async function handleModuleStatus(args: string[]): Promise<CommandResult>
     .where(eq(capabilities.moduleId, moduleId))
     .all();
 
-  // Build sections as multi-line blocks joined by \n\n.
-  // The display code in index.ts splits on \n\n and passes each section
-  // as a single multi-line string to p.log.message(), avoiding clack's
-  // per-line extra spacing.
+  // Build sections as multi-line blocks joined by \n\n. index.ts writes the
+  // whole message to stdout verbatim, so the blank line between sections is
+  // exactly what the operator sees.
   const sections: string[] = [];
 
   // Section 1: Module metadata

package/src/cli/commands/module-update.test.ts

@@ -143,7 +143,7 @@ description: fixture
   });
 
   test('quiet=true suppresses the per-call log lines (caller renders its own)', async () => {
-    // We can't easily intercept @clack's log output without adding test
+    // We can't easily intercept the log helpers' output without adding test
     // hooks, so instead we exercise that the call simply succeeds and
     // returns the structured outcome — the sweep relies on this to
     // render its own output without duplicates. A non-quiet call

package/src/cli/commands/proxmox-template-selection.ts

@@ -60,7 +60,7 @@ function compareDescending(a: ParsedUbuntu, b: ParsedUbuntu): number {
 
 /**
  * Build the option list used by the select prompt and tests.
- * Exported so tests can assert on filtering/sorting without mocking @clack.
+ * Exported so tests can assert on filtering/sorting without driving a prompt.
  */
 export function buildUbuntuOptions(appliances: ProxmoxAppliance[]): ParsedUbuntu[] {
   return appliances

package/src/cli/commands/status.ts

@@ -10,6 +10,7 @@ import { eq } from 'drizzle-orm';
 import { getDb } from '../../db/client';
 import { capabilities, moduleConfigs, modules, systemConfig, systemSecrets } from '../../db/schema';
 import { type ModuleManifest, getSingularSystemSpec } from '../../manifest/schema';
+import { formatPausedDuration } from '../../services/module-pause';
 import type { CommandResult } from '../types';
 
 /**
@@ -26,10 +27,21 @@ async function determineModuleStatus(
   generatedPath: string,
   dbState: string,
 ): Promise<{
-  status: 'IMPORTED' | 'CONFIGURED' | 'GENERATED' | 'DEPLOYED' | 'VERIFIED' | 'NEEDS_UPDATE';
+  status:
+    | 'IMPORTED'
+    | 'CONFIGURED'
+    | 'GENERATED'
+    | 'DEPLOYED'
+    | 'VERIFIED'
+    | 'NEEDS_UPDATE'
+    | 'PAUSED';
   missingCount?: number;
 }> {
-  // Deployed states come directly from the DB — don't infer from filesystem
+  // Deployed states come directly from the DB — don't infer from filesystem.
+  // PAUSED is checked FIRST: a paused module still has a generated/ directory
+  // and a full config, so every derivation below it would report it as an
+  // ordinary deployed module and the pause would be invisible here.
+  if (dbState === 'PAUSED') return { status: 'PAUSED' };
   if (dbState === 'VERIFIED') return { status: 'VERIFIED' };
   if (dbState === 'INSTALLED') return { status: 'DEPLOYED' };
 
@@ -120,6 +132,9 @@ export async function handleStatus(): Promise<CommandResult> {
         );
 
         // Status icon
+        // A paused module is deliberately out of service, so it must never
+        // carry the same ✓ as a healthy one — that is exactly how a pause turns
+        // into an outage nobody notices.
         const icon =
           statusInfo.status === 'VERIFIED' ||
           statusInfo.status === 'DEPLOYED' ||
@@ -128,7 +143,14 @@ export async function handleStatus(): Promise<CommandResult> {
             : '⚠';
 
         lines.push(`  ${icon} ${module.id} (v${module.version})`);
-        lines.push(`    Status: ${statusInfo.status}`);
+        if (statusInfo.status === 'PAUSED') {
+          // State plus AGE — a pause with alerting suppressed is only safe if
+          // how long it has run is impossible to miss (design D7).
+          lines.push(`    Status: PAUSED (${formatPausedDuration(module.pausedAt)})`);
+          if (module.pauseReason) lines.push(`    Paused: ${module.pauseReason}`);
+        } else {
+          lines.push(`    Status: ${statusInfo.status}`);
+        }
 
         // Type: VPS or Container
         const zone = getSingularSystemSpec(manifest)?.zone;

package/src/cli/completion.ts

@@ -194,6 +194,8 @@ export async function getCompletions(words: string[], current: number): Promise<
       'terraform-unlock',
       'types',
       'validate',
+      'pause',
+      'unpause',
     ];
     return filterSuggestions(subcommands, args[1] || '');
   }
@@ -437,6 +439,8 @@ export async function getCompletions(words: string[], current: number): Promise<
       'terraform-unlock',
       'verify',
       'audit', // deprecation alias for `verify`
+      'pause',
+      'unpause',
     ];
     if (moduleCommands.includes(args[1] || '')) {
       const db = getDb();

package/src/cli/fuel-gauge.ts

@@ -10,7 +10,7 @@
  */
 
 import { stdout } from 'node:process';
-import * as p from '@clack/prompts';
+import { log } from '@celilo/cli-display';
 import { getActiveDisplay } from './prompts';
 
 /**
@@ -173,7 +173,7 @@ export class FuelGauge {
     if (!this.running) return;
 
     this.cleanup();
-    p.log.info(`${this.title} (backgrounded)`);
+    log.info(`${this.title} (backgrounded)`);
 
     if (this.onBackground) {
       this.onBackground();
@@ -208,9 +208,9 @@ export class FuelGauge {
     this.cleanup();
 
     if (success) {
-      p.log.success(this.title);
+      log.success(this.title);
     } else {
-      p.log.error(this.title);
+      log.error(this.title);
       console.log('');
       console.log(colors.dim('Last output:'));
       const errorLines = this.outputLines.slice(-this.errorDisplayLines);