@celilo/cli 0.21.0 → 0.23.0

package/src/cli/commands/events.ts

@@ -10,6 +10,7 @@
  *   tail                 recent events
  *   list-subscribers     persistent subscribers
  *   list-pending         pending deliveries (subscriber fan-out — NOT questions)
+ *   list-failed          failed/abandoned deliveries + a TRUE total
  *   list-unanswered      interview questions nobody has answered yet
  *   drain                process pending deliveries once
  *   run                  long-running dispatcher (foreground; SIGINT to stop)
@@ -24,7 +25,9 @@
 import { spawnSync } from 'node:child_process';
 import {
   BUS_VERSION,
+  type FailedBySubscriber,
   defineEvents,
+  describeError,
   drainOnce,
   openBus,
   recoverFromCrash,
@@ -52,7 +55,7 @@ import {
   unitInstalledInAnyScope,
 } from '../../services/events-daemon';
 import { getArg, hasFlag } from '../parser';
-import type { CommandResult } from '../types';
+import type { CommandResult, CommandSuccess } from '../types';
 
 const NO_SCHEMAS = defineEvents({});
 
@@ -60,10 +63,16 @@ function openCliBus() {
   return openBus({ dbPath: getEventBusPath(), events: NO_SCHEMAS });
 }
 
+/**
+ * Every `celilo events` JSON verb returns through here, so `rawOutput` is set
+ * once: without it the payload goes through the CLI's decorating renderer and
+ * reaches stdout prefixed and re-wrapped, and no longer parses.
+ */
 function jsonResult(data: unknown): CommandResult {
   return {
     success: true,
     message: JSON.stringify(data, null, 2),
+    rawOutput: true,
     data,
   };
 }
@@ -120,6 +129,26 @@ export async function handleEventsRunHook(args: string[]): Promise<CommandResult
   const module = db.select().from(modules).where(eq(modules.id, moduleId)).get();
   if (!module) return { success: false, error: `Module not found: ${moduleId}` };
 
+  // Quiescence (openspec/changes/module-pause-lifecycle, task 2.1/2.2). Pausing
+  // drops the module's bus subscriptions, so ordinarily nothing reaches here at
+  // all; this is the second line, and it is load-bearing rather than belt-and-
+  // braces. The subscribers table lives in a DIFFERENT database (events.db) to
+  // the module state, so the two can disagree — `events resync-subscriptions`
+  // rebuilds subscribers from celilo.db, a restore starts events.db empty, and
+  // a hand-written row is always possible. Every one of those paths ends here,
+  // where the module's actual state is readable.
+  //
+  // Success, not failure: the event was delivered correctly and the module is
+  // deliberately not listening. Reporting a failure would retry it up to
+  // max_attempts and then surface as an alert about the pause the operator
+  // themselves took.
+  if (module.state === 'PAUSED') {
+    return {
+      success: true,
+      message: `Skipped ${moduleId}.${subName}: module is paused`,
+    };
+  }
+
   const manifest = module.manifestData as ModuleManifest;
   const sub = (manifest.subscriptions ?? []).find((s) => s.name === subName);
   if (!sub) {
@@ -246,6 +275,75 @@ export async function handleEventsListPending(
   }
 }
 
+/** One failed/abandoned delivery, as `events list-failed` reports it. */
+export interface FailedDelivery {
+  eventId: number;
+  eventType: string | null;
+  subscriber: string | null;
+  status: string;
+  attempts: number;
+  finishedAt: number | null;
+  error: string | null;
+}
+
+/** What `events list-failed` returns: the true total, the shape, and a sample. */
+export interface FailedDeliveryReport {
+  total: number;
+  bySubscriber: FailedBySubscriber[];
+  shown: number;
+  deliveries: FailedDelivery[];
+}
+
+/**
+ * `celilo events list-failed` — what has failed or been abandoned, who it was
+ * for, and why.
+ *
+ * The instrument celilo#623 lacked: `failedDeliveries` had exactly one caller
+ * (the doctor) and no operator surface at all, so the only signal was a count
+ * that was really a LIMIT. `total` here is a real COUNT — `deliveries` is the
+ * sample, capped by `--limit`, and `shown` says so.
+ *
+ * This is NOT `list-pending`, which reads deliveries still queued.
+ */
+export async function handleEventsListFailed(
+  _args: string[],
+  flags: Record<string, string | boolean>,
+): Promise<CommandResult> {
+  const bus = openCliBus();
+  try {
+    const subscriber = typeof flags.subscriber === 'string' ? flags.subscriber : undefined;
+    const limit = flags.limit ? Number(flags.limit) : 50;
+    const { total, bySubscriber } = bus.failedDeliveryTotals();
+    // Subscriber/event names live in other tables; the delivery row has ids.
+    const named = new Map(
+      bus.db
+        .query<{ id: number; name: string }, []>('SELECT id, name FROM subscribers')
+        .all()
+        .map((r) => [r.id, r.name]),
+    );
+    const deliveries: FailedDelivery[] = bus.failedDeliveries({ limit, subscriber }).map((d) => ({
+      eventId: d.eventId,
+      eventType: bus.getEvent(d.eventId)?.type ?? null,
+      subscriber: named.get(d.subscriberId) ?? null,
+      status: d.status,
+      attempts: d.attempts,
+      finishedAt: d.finishedAt,
+      error: describeError(d.lastError),
+    }));
+    const report: FailedDeliveryReport = {
+      total: subscriber
+        ? (bySubscriber.find((s) => s.subscriber === subscriber)?.count ?? 0)
+        : total,
+      bySubscriber,
+      shown: deliveries.length,
+      deliveries,
+    };
+    return jsonResult(report);
+  } finally {
+    bus.close();
+  }
+}
+
 /** One unanswered interview question, as `events list-unanswered` reports it. */
 export interface UnansweredInterview {
   eventId: number;
@@ -810,7 +908,7 @@ async function runProgrammaticResponder(
 
   let stopping = false;
   let stopReason: 'idle' | 'max-duration' | 'signal' = 'idle';
-  const buildSummary = (): { success: true; message: string; data: unknown } => {
+  const buildSummary = (): CommandSuccess => {
     if (!stopping) {
       stopping = true;
       handle.close();
@@ -821,7 +919,12 @@ async function runProgrammaticResponder(
       answered: handle.answered(),
       missed: handle.missed(),
     };
-    return { success: true, message: JSON.stringify(summary, null, 2), data: summary };
+    return {
+      success: true,
+      message: JSON.stringify(summary, null, 2),
+      rawOutput: true,
+      data: summary,
+    };
   };
 
   process.on('SIGINT', () => {

package/src/cli/commands/module-deploy.ts

@@ -71,8 +71,8 @@ export async function handleModuleDeploy(
   }
 
   // Success message is emitted by deployModule via the active ProgressDisplay,
-  // so we return an empty message to avoid the top-level clack outro printing
-  // a duplicate line in a different style.
+  // so we return an empty message — index.ts exits without writing anything
+  // more, rather than printing a duplicate line in a different style.
   return {
     success: true,
     message: '',

package/src/cli/commands/module-health.ts

@@ -79,6 +79,7 @@ export async function handleModuleHealth(
     return {
       success: true,
       message: JSON.stringify({ modules: results }, null, 2),
+      rawOutput: true,
     };
   }
 

package/src/cli/commands/module-import.ts

@@ -143,9 +143,9 @@ export async function handleAspectApprovalAfterImport(args: {
     `celilo will run the '${aspect.ansible_role}' Ansible role on every non-api_only system`,
     'in those zones.',
   ].join('\n');
-  // Use process.stderr.write rather than the prompt UI so the
-  // multi-line scope block is preserved verbatim — @clack/prompts
-  // mangles multi-line message arguments.
+  // Written straight to stderr rather than through the prompt UI so the
+  // multi-line scope block is preserved verbatim — a prompt renderer owns
+  // its frame and reflows a multi-line message argument.
   process.stderr.write(`\n${scopeMsg}\n\n`);
 
   const accepted = await withInterviewSession(() =>

package/src/cli/commands/module-list.ts

@@ -3,6 +3,7 @@ import { type DbClient, getDb } from '../../db/client';
 import { modules, modules as modulesTable, monitors } from '../../db/schema';
 import { moduleHealthCell } from '../../services/alerting/format';
 import { loadAllLiveAlerts, summariseByModule } from '../../services/alerting/store';
+import { formatPausedDuration } from '../../services/module-pause';
 import { hasFlag } from '../parser';
 import type { CommandResult } from '../types';
 
@@ -50,7 +51,17 @@ export async function handleModuleList(
   for (const module of moduleRows) {
     const observed = health.get(module.id);
     const healthNote = observed ? `  [${observed}]` : '';
-    lines.push(`${module.id} (v${module.version}) - ${module.state}${healthNote}`);
+    // A pause suppresses the alerting that would otherwise report this module
+    // as down, so the state alone is not enough — the AGE is what separates a
+    // maintenance window from an outage nobody remembers taking (design D7).
+    const stateCell =
+      module.state === 'PAUSED'
+        ? `PAUSED (${formatPausedDuration(module.pausedAt)})`
+        : module.state;
+    lines.push(`${module.id} (v${module.version}) - ${stateCell}${healthNote}`);
+    if (module.state === 'PAUSED' && module.pauseReason) {
+      lines.push(`  Paused: ${module.pauseReason}`);
+    }
     if (module.description) {
       lines.push(`  ${module.description}`);
     }

package/src/cli/commands/module-pause.ts

@@ -0,0 +1,317 @@
+/**
+ * `celilo module pause <id>` / `celilo module unpause <id>`.
+ *
+ * A thin adapter (Rule 10.5): parse flags, build the plan, confirm, execute,
+ * render. All the decisions live in `services/module-pause.ts` (pure planning)
+ * and the injected deps below (the side effects).
+ *
+ * See openspec/changes/module-pause-lifecycle/.
+ */
+
+import { join } from 'node:path';
+import { eq } from 'drizzle-orm';
+import { ProxmoxClient } from '../../api-clients/proxmox';
+import { getModuleStoragePath } from '../../config/paths';
+import { type DbClient, getDb } from '../../db/client';
+import { moduleSystems, modules } from '../../db/schema';
+import type { ModuleManifest } from '../../manifest/schema';
+import { ModuleManifestSchema } from '../../manifest/schema';
+import { askConfirm, withInterviewSession } from '../../services/bus-interview';
+import { getServiceCredentials } from '../../services/container-service';
+import { deployModule } from '../../services/module-deploy';
+import { checkInFlight } from '../../services/module-operations';
+import { completeOperation, failOperation, startOperation } from '../../services/module-operations';
+import {
+  type ExecutionReport,
+  type InfraStopOutcome,
+  type ModuleSnapshot,
+  type PauseDeps,
+  type PausePlan,
+  PauseRefusedError,
+  actedOn,
+  describeMachineStopInfra,
+  executePause,
+  executeUnpause,
+  planPause,
+  planUnpause,
+} from '../../services/module-pause';
+import {
+  registerModuleSubscriptions,
+  unregisterModuleSubscriptions,
+} from '../../services/module-subscriptions';
+import { getArg, hasFlag } from '../parser';
+import { log } from '../prompts';
+import type { CommandResult } from '../types';
+
+/**
+ * Every module celilo knows about, in the shape the planner wants. The planner
+ * is pure, so the whole fleet is read once here rather than queried per step.
+ *
+ * A module whose stored manifest no longer parses is dropped rather than
+ * failing the command: it cannot be a graph node, and refusing to pause the
+ * fleet because one unrelated manifest went stale would be the wrong trade.
+ */
+function loadFleet(db: DbClient): ModuleSnapshot[] {
+  const snapshots: ModuleSnapshot[] = [];
+  for (const row of db.select().from(modules).all()) {
+    const parsed = ModuleManifestSchema.safeParse(row.manifestData);
+    if (!parsed.success) continue;
+    snapshots.push({
+      id: row.id,
+      state: row.state,
+      pausedAt: row.pausedAt,
+      pauseReason: row.pauseReason,
+      manifest: parsed.data,
+    });
+  }
+  return snapshots;
+}
+
+/** moduleId → description, for the in-flight refusal (task 3.8). */
+function inFlightByModule(): Map<string, string> {
+  const map = new Map<string, string>();
+  for (const conflict of checkInFlight()) {
+    map.set(conflict.operation.moduleId, conflict.describe);
+  }
+  return map;
+}
+
+/**
+ * `--stop-infra` (design D2, revised). Opt-in, and deliberately NOT what a pause
+ * means: pausing `caddy` to swap the *firewall* must not take every website
+ * down.
+ *
+ * It acts ONLY on infrastructure celilo provisioned for this module:
+ *   - celilo-provisioned LXC/VM -> stopped; celilo created it, so it is celilo's
+ *   - machine-pool system       -> not applicable, by design (see below)
+ *   - systemless driver         -> nothing to stop
+ *
+ * The flag is a convenience for "I actually want the box off". Pause's real job
+ * is control-plane quiescence, and the provider swap this feature exists for
+ * never needs the flag at all.
+ */
+async function stopModuleInfrastructure(db: DbClient, moduleId: string): Promise<InfraStopOutcome> {
+  const systems = db.select().from(moduleSystems).where(eq(moduleSystems.moduleId, moduleId)).all();
+
+  if (systems.length === 0) {
+    // A driver module (`greenwave`, `axon`, `namecheap`) declares no
+    // `requires.system` — it talks to a device over HTTP. Nothing to stop, and
+    // that is a normal outcome to report, not an error (spec scenario
+    // "Shutdown on a systemless driver module reports nothing to stop").
+    return { stopped: false, detail: 'no infrastructure to stop' };
+  }
+
+  const detail: string[] = [];
+  for (const system of systems) {
+    if (system.infraType === 'machine') {
+      // NOT APPLICABLE, by design — not a missing feature (design D2, revised).
+      //
+      // `--stop-infra` acts only on infrastructure celilo PROVISIONED for the
+      // module. A machine-pool system is operator-pre-provisioned: it may
+      // predate celilo, and it may run things celilo has never heard of — not
+      // merely other celilo modules, but arbitrary operator work. Powering it
+      // off, or stopping services on it, reaches outside what celilo owns.
+      //
+      // Same principle as the sizing rule: a module must not own a host-level
+      // fact, because the host outlives any one module's config. Framing this
+      // as "celilo cannot identify the service unit" would be wrong — it
+      // implies a capability gap, when the answer is that this is not celilo's
+      // to stop.
+      detail.push(describeMachineStopInfra(system.hostname, moduleId));
+      continue;
+    }
+
+    if (system.vmid == null || !system.serviceId) {
+      detail.push(`${system.hostname}: no container recorded, nothing to stop`);
+      continue;
+    }
+
+    const credentials = await getServiceCredentials(system.serviceId);
+    if (!('api_url' in credentials)) {
+      detail.push(`${system.hostname}: container service is not Proxmox, nothing to stop`);
+      continue;
+    }
+
+    const client = new ProxmoxClient(credentials);
+    const result = await client.setGuestPower(system.vmid, 'lxc', 'shutdown');
+    detail.push(
+      result.success
+        ? `stopped ${system.hostname} (vmid ${system.vmid})`
+        : `could not stop ${system.hostname} (vmid ${system.vmid}): ${result.message}`,
+    );
+  }
+
+  return { stopped: true, detail: detail.join('; ') };
+}
+
+function buildDeps(db: DbClient): PauseDeps {
+  return {
+    db,
+    unsubscribe: (moduleId) => {
+      // The primary quiescence mechanism: with no subscriber rows the
+      // dispatcher has nothing to deliver to. `run-named-hook` guards the
+      // paths that do not go through the bus.
+      unregisterModuleSubscriptions(moduleId);
+    },
+    resubscribe: (moduleId) => {
+      // A deploy does NOT re-register subscriptions (only import and
+      // `module update` do), so unpause has to — otherwise the module comes
+      // back deployed but permanently deaf.
+      const row = db.select().from(modules).where(eq(modules.id, moduleId)).get();
+      if (!row) return;
+      const manifest = row.manifestData as ModuleManifest;
+      registerModuleSubscriptions(manifest, join(getModuleStoragePath(), moduleId));
+    },
+    redeploy: async (moduleId) => {
+      try {
+        const result = await deployModule(moduleId, db, {});
+        return { success: result.success, error: result.error };
+      } catch (err) {
+        return { success: false, error: err instanceof Error ? err.message : String(err) };
+      }
+    },
+    stopInfrastructure: (moduleId) => stopModuleInfrastructure(db, moduleId),
+    startOperation,
+    completeOperation,
+    failOperation,
+    now: () => new Date(),
+    log: (message) => log.info(message),
+  };
+}
+
+/** The ordered plan, rendered for `--dry-run` and for the confirmation. */
+function renderPlan(plan: PausePlan): string {
+  const verb = plan.action === 'pause' ? 'Pause' : 'Unpause';
+  const order = plan.action === 'pause' ? 'consumers first' : 'providers first';
+  const lines = [
+    plan.cascade
+      ? `${verb} ${plan.requested} and its transitive consumers (${order}):`
+      : `${verb} ${plan.requested}:`,
+  ];
+  plan.steps.forEach((step, index) => {
+    const suffix = step.disposition === 'act' ? '' : ` — skip (${step.note})`;
+    lines.push(`  ${index + 1}. ${step.moduleId}${suffix}`);
+  });
+  if (plan.stopInfra) {
+    lines.push('', 'Infrastructure will also be stopped (--stop-infra).');
+  }
+  return lines.join('\n');
+}
+
+function renderReport(report: ExecutionReport): string {
+  const lines = report.outcomes.map((o) => {
+    const mark = o.result === 'acted' ? '✓' : o.result === 'skipped' ? '·' : '✗';
+    return `  ${mark} ${o.moduleId}${o.detail ? ` — ${o.detail}` : ''}`;
+  });
+  const acted = report.outcomes.filter((o) => o.result === 'acted').length;
+  const skipped = report.outcomes.filter((o) => o.result === 'skipped').length;
+  const failed = report.outcomes.filter((o) => o.result === 'failed').length;
+  lines.push('', `${acted} changed, ${skipped} already done, ${failed} failed.`);
+  return lines.join('\n');
+}
+
+/**
+ * Confirmation for a cascade. An event-bus interview question, never a stdin
+ * prompt (design D6), so the operation is drivable headlessly — by CI, by the
+ * MCP, by a remote responder. `--yes` satisfies it without asking.
+ */
+async function confirmCascade(plan: PausePlan, yes: boolean): Promise<boolean> {
+  if (!plan.cascade || yes) return true;
+  const affected = actedOn(plan);
+  if (affected.length === 0) return true;
+
+  return withInterviewSession(() =>
+    askConfirm({
+      scope: `module-${plan.action}:${plan.requested}`,
+      key: 'cascade',
+      message: `${plan.action === 'pause' ? 'Pause' : 'Unpause'} ${affected.length} module(s): ${affected.join(', ')}?`,
+      description: renderPlan(plan),
+      defaultValue: false,
+    }),
+  );
+}
+
+export async function handleModulePause(
+  args: string[],
+  flags: Record<string, string | boolean> = {},
+): Promise<CommandResult> {
+  const moduleId = getArg(args, 0);
+  if (!moduleId) {
+    return {
+      success: false,
+      error:
+        'Module ID is required\n\nUsage: celilo module pause <id> [--cascade] [--stop-infra] [--dry-run] [--yes] [--reason "..."]',
+    };
+  }
+
+  const db = getDb();
+  const reason = typeof flags.reason === 'string' ? flags.reason : null;
+
+  let plan: PausePlan;
+  try {
+    plan = planPause({
+      moduleId,
+      fleet: loadFleet(db),
+      cascade: hasFlag(flags, 'cascade'),
+      stopInfra: hasFlag(flags, 'stop-infra'),
+      inFlight: inFlightByModule(),
+    });
+  } catch (err) {
+    if (err instanceof PauseRefusedError) return { success: false, error: err.message };
+    throw err;
+  }
+
+  if (hasFlag(flags, 'dry-run')) {
+    return { success: true, message: `${renderPlan(plan)}\n\n(dry run — nothing was changed)` };
+  }
+
+  if (!(await confirmCascade(plan, hasFlag(flags, 'yes')))) {
+    return { success: false, error: 'Cancelled — nothing was paused' };
+  }
+
+  const report = await executePause(plan, buildDeps(db), reason);
+  return report.success
+    ? { success: true, message: renderReport(report) }
+    : { success: false, error: renderReport(report) };
+}
+
+export async function handleModuleUnpause(
+  args: string[],
+  flags: Record<string, string | boolean> = {},
+): Promise<CommandResult> {
+  const moduleId = getArg(args, 0);
+  if (!moduleId) {
+    return {
+      success: false,
+      error:
+        'Module ID is required\n\nUsage: celilo module unpause <id> [--cascade] [--dry-run] [--yes]',
+    };
+  }
+
+  const db = getDb();
+
+  let plan: PausePlan;
+  try {
+    plan = planUnpause({
+      moduleId,
+      fleet: loadFleet(db),
+      cascade: hasFlag(flags, 'cascade'),
+    });
+  } catch (err) {
+    if (err instanceof PauseRefusedError) return { success: false, error: err.message };
+    throw err;
+  }
+
+  if (hasFlag(flags, 'dry-run')) {
+    return { success: true, message: `${renderPlan(plan)}\n\n(dry run — nothing was changed)` };
+  }
+
+  if (!(await confirmCascade(plan, hasFlag(flags, 'yes')))) {
+    return { success: false, error: 'Cancelled — nothing was unpaused' };
+  }
+
+  const report = await executeUnpause(plan, buildDeps(db));
+  return report.success
+    ? { success: true, message: renderReport(report) }
+    : { success: false, error: renderReport(report) };
+}