@celilo/cli 0.22.0 → 0.24.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (104) hide show
  1. package/CELILO_CORE_MODULES.md +2 -2
  2. package/CELILO_SUBSYSTEMS.md +61 -9
  3. package/drizzle/0024_module_pause.sql +20 -0
  4. package/drizzle/meta/_journal.json +8 -1
  5. package/package.json +7 -7
  6. package/src/__integration__/container-services-cli.integration.test.ts +8 -2
  7. package/src/api/remote-client.test.ts +6 -5
  8. package/src/api/serve.ts +41 -7
  9. package/src/api-clients/proxmox.ts +34 -0
  10. package/src/cli/commands/alerts-act.ts +1 -1
  11. package/src/cli/commands/alerts-sweep.ts +2 -0
  12. package/src/cli/commands/backup-create.ts +26 -11
  13. package/src/cli/commands/backup-list.test.ts +83 -0
  14. package/src/cli/commands/backup-list.ts +67 -3
  15. package/src/cli/commands/backup-prune.ts +17 -17
  16. package/src/cli/commands/backup-sweep.ts +20 -8
  17. package/src/cli/commands/events.ts +34 -3
  18. package/src/cli/commands/firewall-interface-list.test.ts +85 -0
  19. package/src/cli/commands/firewall-interface-list.ts +123 -0
  20. package/src/cli/commands/machine-add.ts +30 -2
  21. package/src/cli/commands/module-config.test.ts +64 -2
  22. package/src/cli/commands/module-config.ts +159 -8
  23. package/src/cli/commands/module-deploy.ts +2 -2
  24. package/src/cli/commands/module-health.ts +1 -0
  25. package/src/cli/commands/module-import.ts +3 -3
  26. package/src/cli/commands/module-list.ts +12 -1
  27. package/src/cli/commands/module-pause.ts +317 -0
  28. package/src/cli/commands/module-remove.ts +78 -40
  29. package/src/cli/commands/module-status.ts +127 -4
  30. package/src/cli/commands/module-update.test.ts +1 -1
  31. package/src/cli/commands/monitor.ts +116 -19
  32. package/src/cli/commands/proxmox-template-selection.ts +1 -1
  33. package/src/cli/commands/status.ts +25 -3
  34. package/src/cli/commands/system-migrate.ts +14 -0
  35. package/src/cli/commands/system-update.ts +4 -1
  36. package/src/cli/completion.ts +39 -9
  37. package/src/cli/fuel-gauge.ts +4 -4
  38. package/src/cli/index.ts +104 -22
  39. package/src/cli/json-output.test.ts +162 -0
  40. package/src/cli/prompts.ts +53 -74
  41. package/src/cli/service-credential.ts +3 -3
  42. package/src/cli/stdout-is-undecorated.test.ts +94 -0
  43. package/src/cli/tui/audit-state.ts +2 -0
  44. package/src/cli/types.ts +7 -2
  45. package/src/db/schema.ts +73 -15
  46. package/src/hooks/capability-loader.ts +130 -4
  47. package/src/hooks/run-named-hook.ts +28 -0
  48. package/src/hooks/types.ts +2 -1
  49. package/src/manifest/contracts/v1.ts +16 -0
  50. package/src/manifest/schema.ts +40 -65
  51. package/src/services/alerting/builtin-monitors.test.ts +18 -10
  52. package/src/services/alerting/cadence-migration.test.ts +155 -0
  53. package/src/services/alerting/cadence-migration.ts +90 -0
  54. package/src/services/alerting/coverage-source.ts +8 -11
  55. package/src/services/alerting/deploy-hooks.test.ts +16 -7
  56. package/src/services/alerting/deploy-hooks.ts +11 -5
  57. package/src/services/alerting/health-cadence.test.ts +58 -0
  58. package/src/services/alerting/health-cadence.ts +128 -0
  59. package/src/services/alerting/health-coverage.ts +18 -8
  60. package/src/services/alerting/monitors.ts +50 -15
  61. package/src/services/alerting/suppression.test.ts +5 -0
  62. package/src/services/alerting/suppression.ts +18 -1
  63. package/src/services/alerting/sweep-runner.test.ts +52 -3
  64. package/src/services/alerting/sweep-runner.ts +41 -8
  65. package/src/services/audit/backup-source.ts +24 -1
  66. package/src/services/audit/backups.test.ts +95 -10
  67. package/src/services/audit/backups.ts +40 -37
  68. package/src/services/audit/interface-classification.test.ts +220 -0
  69. package/src/services/audit/interface-classification.ts +167 -0
  70. package/src/services/audit/types.ts +2 -1
  71. package/src/services/backup-age-agreement.test.ts +118 -0
  72. package/src/services/backup-create.ts +36 -30
  73. package/src/services/backup-metadata.ts +52 -1
  74. package/src/services/backup-retention.test.ts +123 -0
  75. package/src/services/backup-retention.ts +66 -5
  76. package/src/services/backup-schedule.test.ts +166 -0
  77. package/src/services/backup-schedule.ts +105 -15
  78. package/src/services/backup-staging.ts +14 -1
  79. package/src/services/backup-sweep.test.ts +22 -3
  80. package/src/services/backup-sweep.ts +15 -5
  81. package/src/services/bus-interview.ts +2 -2
  82. package/src/services/bus-secret-flow.test.ts +1 -1
  83. package/src/services/cadence.test.ts +97 -0
  84. package/src/services/cadence.ts +165 -0
  85. package/src/services/fleet-checks.ts +48 -0
  86. package/src/services/machine-detector.ts +23 -1
  87. package/src/services/module-config.ts +33 -0
  88. package/src/services/module-deploy.ts +1 -1
  89. package/src/services/module-pause-observability.test.ts +224 -0
  90. package/src/services/module-pause-quiescence.test.ts +163 -0
  91. package/src/services/module-pause.test.ts +573 -0
  92. package/src/services/module-pause.ts +544 -0
  93. package/src/services/remove-guard.test.ts +175 -0
  94. package/src/services/remove-guard.ts +109 -0
  95. package/src/services/storage-providers/s3.test.ts +96 -13
  96. package/src/services/storage-providers/s3.ts +48 -15
  97. package/src/services/terminal-responder.ts +16 -16
  98. package/src/services/update/dep-graph.test.ts +33 -4
  99. package/src/services/update/dep-graph.ts +39 -17
  100. package/src/services/zone-detector.test.ts +34 -3
  101. package/src/services/zone-detector.ts +32 -49
  102. package/src/test-utils/cli.ts +15 -14
  103. package/src/test-utils/integration-guard.ts +26 -0
  104. package/src/test-utils/setup-test-db.ts +13 -23
@@ -0,0 +1,317 @@
1
+ /**
2
+ * `celilo module pause <id>` / `celilo module unpause <id>`.
3
+ *
4
+ * A thin adapter (Rule 10.5): parse flags, build the plan, confirm, execute,
5
+ * render. All the decisions live in `services/module-pause.ts` (pure planning)
6
+ * and the injected deps below (the side effects).
7
+ *
8
+ * See openspec/changes/module-pause-lifecycle/.
9
+ */
10
+
11
+ import { join } from 'node:path';
12
+ import { eq } from 'drizzle-orm';
13
+ import { ProxmoxClient } from '../../api-clients/proxmox';
14
+ import { getModuleStoragePath } from '../../config/paths';
15
+ import { type DbClient, getDb } from '../../db/client';
16
+ import { moduleSystems, modules } from '../../db/schema';
17
+ import type { ModuleManifest } from '../../manifest/schema';
18
+ import { ModuleManifestSchema } from '../../manifest/schema';
19
+ import { askConfirm, withInterviewSession } from '../../services/bus-interview';
20
+ import { getServiceCredentials } from '../../services/container-service';
21
+ import { deployModule } from '../../services/module-deploy';
22
+ import { checkInFlight } from '../../services/module-operations';
23
+ import { completeOperation, failOperation, startOperation } from '../../services/module-operations';
24
+ import {
25
+ type ExecutionReport,
26
+ type InfraStopOutcome,
27
+ type ModuleSnapshot,
28
+ type PauseDeps,
29
+ type PausePlan,
30
+ PauseRefusedError,
31
+ actedOn,
32
+ describeMachineStopInfra,
33
+ executePause,
34
+ executeUnpause,
35
+ planPause,
36
+ planUnpause,
37
+ } from '../../services/module-pause';
38
+ import {
39
+ registerModuleSubscriptions,
40
+ unregisterModuleSubscriptions,
41
+ } from '../../services/module-subscriptions';
42
+ import { getArg, hasFlag } from '../parser';
43
+ import { log } from '../prompts';
44
+ import type { CommandResult } from '../types';
45
+
46
+ /**
47
+ * Every module celilo knows about, in the shape the planner wants. The planner
48
+ * is pure, so the whole fleet is read once here rather than queried per step.
49
+ *
50
+ * A module whose stored manifest no longer parses is dropped rather than
51
+ * failing the command: it cannot be a graph node, and refusing to pause the
52
+ * fleet because one unrelated manifest went stale would be the wrong trade.
53
+ */
54
+ function loadFleet(db: DbClient): ModuleSnapshot[] {
55
+ const snapshots: ModuleSnapshot[] = [];
56
+ for (const row of db.select().from(modules).all()) {
57
+ const parsed = ModuleManifestSchema.safeParse(row.manifestData);
58
+ if (!parsed.success) continue;
59
+ snapshots.push({
60
+ id: row.id,
61
+ state: row.state,
62
+ pausedAt: row.pausedAt,
63
+ pauseReason: row.pauseReason,
64
+ manifest: parsed.data,
65
+ });
66
+ }
67
+ return snapshots;
68
+ }
69
+
70
+ /** moduleId → description, for the in-flight refusal (task 3.8). */
71
+ function inFlightByModule(): Map<string, string> {
72
+ const map = new Map<string, string>();
73
+ for (const conflict of checkInFlight()) {
74
+ map.set(conflict.operation.moduleId, conflict.describe);
75
+ }
76
+ return map;
77
+ }
78
+
79
+ /**
80
+ * `--stop-infra` (design D2, revised). Opt-in, and deliberately NOT what a pause
81
+ * means: pausing `caddy` to swap the *firewall* must not take every website
82
+ * down.
83
+ *
84
+ * It acts ONLY on infrastructure celilo provisioned for this module:
85
+ * - celilo-provisioned LXC/VM -> stopped; celilo created it, so it is celilo's
86
+ * - machine-pool system -> not applicable, by design (see below)
87
+ * - systemless driver -> nothing to stop
88
+ *
89
+ * The flag is a convenience for "I actually want the box off". Pause's real job
90
+ * is control-plane quiescence, and the provider swap this feature exists for
91
+ * never needs the flag at all.
92
+ */
93
+ async function stopModuleInfrastructure(db: DbClient, moduleId: string): Promise<InfraStopOutcome> {
94
+ const systems = db.select().from(moduleSystems).where(eq(moduleSystems.moduleId, moduleId)).all();
95
+
96
+ if (systems.length === 0) {
97
+ // A driver module (`greenwave`, `axon`, `namecheap`) declares no
98
+ // `requires.system` — it talks to a device over HTTP. Nothing to stop, and
99
+ // that is a normal outcome to report, not an error (spec scenario
100
+ // "Shutdown on a systemless driver module reports nothing to stop").
101
+ return { stopped: false, detail: 'no infrastructure to stop' };
102
+ }
103
+
104
+ const detail: string[] = [];
105
+ for (const system of systems) {
106
+ if (system.infraType === 'machine') {
107
+ // NOT APPLICABLE, by design — not a missing feature (design D2, revised).
108
+ //
109
+ // `--stop-infra` acts only on infrastructure celilo PROVISIONED for the
110
+ // module. A machine-pool system is operator-pre-provisioned: it may
111
+ // predate celilo, and it may run things celilo has never heard of — not
112
+ // merely other celilo modules, but arbitrary operator work. Powering it
113
+ // off, or stopping services on it, reaches outside what celilo owns.
114
+ //
115
+ // Same principle as the sizing rule: a module must not own a host-level
116
+ // fact, because the host outlives any one module's config. Framing this
117
+ // as "celilo cannot identify the service unit" would be wrong — it
118
+ // implies a capability gap, when the answer is that this is not celilo's
119
+ // to stop.
120
+ detail.push(describeMachineStopInfra(system.hostname, moduleId));
121
+ continue;
122
+ }
123
+
124
+ if (system.vmid == null || !system.serviceId) {
125
+ detail.push(`${system.hostname}: no container recorded, nothing to stop`);
126
+ continue;
127
+ }
128
+
129
+ const credentials = await getServiceCredentials(system.serviceId);
130
+ if (!('api_url' in credentials)) {
131
+ detail.push(`${system.hostname}: container service is not Proxmox, nothing to stop`);
132
+ continue;
133
+ }
134
+
135
+ const client = new ProxmoxClient(credentials);
136
+ const result = await client.setGuestPower(system.vmid, 'lxc', 'shutdown');
137
+ detail.push(
138
+ result.success
139
+ ? `stopped ${system.hostname} (vmid ${system.vmid})`
140
+ : `could not stop ${system.hostname} (vmid ${system.vmid}): ${result.message}`,
141
+ );
142
+ }
143
+
144
+ return { stopped: true, detail: detail.join('; ') };
145
+ }
146
+
147
+ function buildDeps(db: DbClient): PauseDeps {
148
+ return {
149
+ db,
150
+ unsubscribe: (moduleId) => {
151
+ // The primary quiescence mechanism: with no subscriber rows the
152
+ // dispatcher has nothing to deliver to. `run-named-hook` guards the
153
+ // paths that do not go through the bus.
154
+ unregisterModuleSubscriptions(moduleId);
155
+ },
156
+ resubscribe: (moduleId) => {
157
+ // A deploy does NOT re-register subscriptions (only import and
158
+ // `module update` do), so unpause has to — otherwise the module comes
159
+ // back deployed but permanently deaf.
160
+ const row = db.select().from(modules).where(eq(modules.id, moduleId)).get();
161
+ if (!row) return;
162
+ const manifest = row.manifestData as ModuleManifest;
163
+ registerModuleSubscriptions(manifest, join(getModuleStoragePath(), moduleId));
164
+ },
165
+ redeploy: async (moduleId) => {
166
+ try {
167
+ const result = await deployModule(moduleId, db, {});
168
+ return { success: result.success, error: result.error };
169
+ } catch (err) {
170
+ return { success: false, error: err instanceof Error ? err.message : String(err) };
171
+ }
172
+ },
173
+ stopInfrastructure: (moduleId) => stopModuleInfrastructure(db, moduleId),
174
+ startOperation,
175
+ completeOperation,
176
+ failOperation,
177
+ now: () => new Date(),
178
+ log: (message) => log.info(message),
179
+ };
180
+ }
181
+
182
+ /** The ordered plan, rendered for `--dry-run` and for the confirmation. */
183
+ function renderPlan(plan: PausePlan): string {
184
+ const verb = plan.action === 'pause' ? 'Pause' : 'Unpause';
185
+ const order = plan.action === 'pause' ? 'consumers first' : 'providers first';
186
+ const lines = [
187
+ plan.cascade
188
+ ? `${verb} ${plan.requested} and its transitive consumers (${order}):`
189
+ : `${verb} ${plan.requested}:`,
190
+ ];
191
+ plan.steps.forEach((step, index) => {
192
+ const suffix = step.disposition === 'act' ? '' : ` — skip (${step.note})`;
193
+ lines.push(` ${index + 1}. ${step.moduleId}${suffix}`);
194
+ });
195
+ if (plan.stopInfra) {
196
+ lines.push('', 'Infrastructure will also be stopped (--stop-infra).');
197
+ }
198
+ return lines.join('\n');
199
+ }
200
+
201
+ function renderReport(report: ExecutionReport): string {
202
+ const lines = report.outcomes.map((o) => {
203
+ const mark = o.result === 'acted' ? '✓' : o.result === 'skipped' ? '·' : '✗';
204
+ return ` ${mark} ${o.moduleId}${o.detail ? ` — ${o.detail}` : ''}`;
205
+ });
206
+ const acted = report.outcomes.filter((o) => o.result === 'acted').length;
207
+ const skipped = report.outcomes.filter((o) => o.result === 'skipped').length;
208
+ const failed = report.outcomes.filter((o) => o.result === 'failed').length;
209
+ lines.push('', `${acted} changed, ${skipped} already done, ${failed} failed.`);
210
+ return lines.join('\n');
211
+ }
212
+
213
+ /**
214
+ * Confirmation for a cascade. An event-bus interview question, never a stdin
215
+ * prompt (design D6), so the operation is drivable headlessly — by CI, by the
216
+ * MCP, by a remote responder. `--yes` satisfies it without asking.
217
+ */
218
+ async function confirmCascade(plan: PausePlan, yes: boolean): Promise<boolean> {
219
+ if (!plan.cascade || yes) return true;
220
+ const affected = actedOn(plan);
221
+ if (affected.length === 0) return true;
222
+
223
+ return withInterviewSession(() =>
224
+ askConfirm({
225
+ scope: `module-${plan.action}:${plan.requested}`,
226
+ key: 'cascade',
227
+ message: `${plan.action === 'pause' ? 'Pause' : 'Unpause'} ${affected.length} module(s): ${affected.join(', ')}?`,
228
+ description: renderPlan(plan),
229
+ defaultValue: false,
230
+ }),
231
+ );
232
+ }
233
+
234
+ export async function handleModulePause(
235
+ args: string[],
236
+ flags: Record<string, string | boolean> = {},
237
+ ): Promise<CommandResult> {
238
+ const moduleId = getArg(args, 0);
239
+ if (!moduleId) {
240
+ return {
241
+ success: false,
242
+ error:
243
+ 'Module ID is required\n\nUsage: celilo module pause <id> [--cascade] [--stop-infra] [--dry-run] [--yes] [--reason "..."]',
244
+ };
245
+ }
246
+
247
+ const db = getDb();
248
+ const reason = typeof flags.reason === 'string' ? flags.reason : null;
249
+
250
+ let plan: PausePlan;
251
+ try {
252
+ plan = planPause({
253
+ moduleId,
254
+ fleet: loadFleet(db),
255
+ cascade: hasFlag(flags, 'cascade'),
256
+ stopInfra: hasFlag(flags, 'stop-infra'),
257
+ inFlight: inFlightByModule(),
258
+ });
259
+ } catch (err) {
260
+ if (err instanceof PauseRefusedError) return { success: false, error: err.message };
261
+ throw err;
262
+ }
263
+
264
+ if (hasFlag(flags, 'dry-run')) {
265
+ return { success: true, message: `${renderPlan(plan)}\n\n(dry run — nothing was changed)` };
266
+ }
267
+
268
+ if (!(await confirmCascade(plan, hasFlag(flags, 'yes')))) {
269
+ return { success: false, error: 'Cancelled — nothing was paused' };
270
+ }
271
+
272
+ const report = await executePause(plan, buildDeps(db), reason);
273
+ return report.success
274
+ ? { success: true, message: renderReport(report) }
275
+ : { success: false, error: renderReport(report) };
276
+ }
277
+
278
+ export async function handleModuleUnpause(
279
+ args: string[],
280
+ flags: Record<string, string | boolean> = {},
281
+ ): Promise<CommandResult> {
282
+ const moduleId = getArg(args, 0);
283
+ if (!moduleId) {
284
+ return {
285
+ success: false,
286
+ error:
287
+ 'Module ID is required\n\nUsage: celilo module unpause <id> [--cascade] [--dry-run] [--yes]',
288
+ };
289
+ }
290
+
291
+ const db = getDb();
292
+
293
+ let plan: PausePlan;
294
+ try {
295
+ plan = planUnpause({
296
+ moduleId,
297
+ fleet: loadFleet(db),
298
+ cascade: hasFlag(flags, 'cascade'),
299
+ });
300
+ } catch (err) {
301
+ if (err instanceof PauseRefusedError) return { success: false, error: err.message };
302
+ throw err;
303
+ }
304
+
305
+ if (hasFlag(flags, 'dry-run')) {
306
+ return { success: true, message: `${renderPlan(plan)}\n\n(dry run — nothing was changed)` };
307
+ }
308
+
309
+ if (!(await confirmCascade(plan, hasFlag(flags, 'yes')))) {
310
+ return { success: false, error: 'Cancelled — nothing was unpaused' };
311
+ }
312
+
313
+ const report = await executeUnpause(plan, buildDeps(db));
314
+ return report.success
315
+ ? { success: true, message: renderReport(report) }
316
+ : { success: false, error: renderReport(report) };
317
+ }
@@ -24,6 +24,11 @@ import {
24
24
  } from '../../services/celilo-events';
25
25
  import { getContainerService, getServiceCredentials } from '../../services/container-service';
26
26
  import { completeOperation, failOperation, startOperation } from '../../services/module-operations';
27
+ import {
28
+ type DependentCandidate,
29
+ describeRemovalRefusal,
30
+ findRemovalBlockers,
31
+ } from '../../services/remove-guard';
27
32
  import { cleanupWebRoutesForModule } from '../../services/web-route-cleanup';
28
33
  import { getArg, hasFlag, validateRequiredArgs } from '../parser';
29
34
  import { log } from '../prompts';
@@ -81,23 +86,26 @@ export async function handleModuleRemove(
81
86
  .all();
82
87
 
83
88
  if (providedCapabilities.length > 0) {
84
- const providedNames = new Set(providedCapabilities.map((c) => c.capabilityName));
89
+ const providedNames = providedCapabilities.map((c) => c.capabilityName);
85
90
  const otherModules = db.select().from(modules).where(ne(modules.id, moduleId)).all();
86
91
 
87
- const dependents = otherModules
88
- .filter((m) => {
89
- const parsed = ModuleManifestSchema.safeParse(m.manifestData);
90
- if (!parsed.success) return false;
91
- return (parsed.data.requires?.capabilities ?? []).some((req) =>
92
- providedNames.has(req.name),
93
- );
94
- })
95
- .map((m) => m.id);
96
-
97
- if (dependents.length > 0) {
92
+ const candidates: DependentCandidate[] = [];
93
+ for (const m of otherModules) {
94
+ const parsed = ModuleManifestSchema.safeParse(m.manifestData);
95
+ if (!parsed.success) continue;
96
+ candidates.push({
97
+ id: m.id,
98
+ manifest: parsed.data,
99
+ paused: m.state === 'PAUSED',
100
+ deployed: !(['IMPORTED', 'VALIDATED', 'CONFIGURED'] as string[]).includes(m.state),
101
+ });
102
+ }
103
+
104
+ const blockers = findRemovalBlockers(providedNames, candidates);
105
+ if (blockers.length > 0) {
98
106
  return {
99
107
  success: false,
100
- error: `Cannot remove '${moduleId}': the following installed modules depend on its capabilities: ${dependents.join(', ')}`,
108
+ error: describeRemovalRefusal(moduleId, blockers),
101
109
  };
102
110
  }
103
111
  }
@@ -163,9 +171,9 @@ async function performModuleRemove(
163
171
  // prompt on hook failure.
164
172
  const manifestForHook = module.manifestData as { hooks?: { on_uninstall?: unknown } } | undefined;
165
173
  if (manifestForHook?.hooks?.on_uninstall) {
166
- // Match build-stream's TTY detection: in non-TTY contexts (tests,
167
- // pipes, CI) skipAnimation prevents the gauge's setInterval and
168
- // raw-stdin handlers from blocking process exit.
174
+ // Match build-stream's TTY detection: in non-TTY contexts (tests, pipes,
175
+ // CI) skipAnimation prevents the gauge's setInterval and raw-stdin
176
+ // handlers from blocking process exit.
169
177
  const isInteractive = process.stdout.isTTY && process.stdin.isTTY;
170
178
  const gauge = new FuelGauge(`${moduleId}: on_uninstall`, {
171
179
  skipAnimation: !isInteractive,
@@ -173,35 +181,65 @@ async function performModuleRemove(
173
181
  gauge.start();
174
182
  const logger = createGaugeLogger(gauge, moduleId, 'on_uninstall');
175
183
  const hookResult = await runNamedHook(moduleId, 'on_uninstall', db, logger, {});
184
+
176
185
  if (hookResult.success) {
177
186
  gauge.stop(true);
178
187
  } else {
179
188
  gauge.stop(false);
180
- log.warn(`on_uninstall failed: ${hookResult.error ?? 'unknown error'}`);
181
- if (force) {
182
- log.info('--force set, continuing removal despite hook failure');
183
- } else if (!isInteractive) {
184
- // Non-TTY (tests, scripts, piped input). Don't try to prompt —
185
- // @clack/prompts hangs on stdin in that context. Default
186
- // behaviour: continue with removal so a hook crash doesn't
187
- // wedge automation. Operators who want strict failure can
188
- // re-run `celilo module run-hook <id> on_uninstall` manually.
189
- log.warn('Non-interactive context detected; continuing removal despite hook failure');
190
- } else {
191
- const proceed = await withInterviewSession(() =>
192
- askConfirm({
193
- scope: `module-remove:${moduleId}`,
194
- key: 'continue_after_hook_failure',
195
- message:
196
- 'on_uninstall hook failed. Continue removing the module anyway? ' +
197
- '(some external state may not be cleaned up)',
198
- defaultValue: false,
199
- }),
200
- );
201
- if (!proceed) {
202
- return { success: false, error: 'Removal cancelled after on_uninstall failure' };
203
- }
189
+ const reason = hookResult.error ?? 'unknown error';
190
+
191
+ if (!force) {
192
+ // STOP, and leave the module ERRORED. Do not ask.
193
+ //
194
+ // `on_uninstall` is what withdraws a module's cross-module state —
195
+ // caddy unexposes its ports and clears its Caddyfile, a firewall
196
+ // provider withdraws its forwards. If it failed, that state is still
197
+ // out there and we do not know how much of it the hook managed before
198
+ // dying. Deleting the module now orphans whatever is left, with no
199
+ // record of what to go clean up.
200
+ //
201
+ // This previously asked "continue removing anyway?" in a TTY and
202
+ // silently continued when not one. Both were wrong. The prompt offers
203
+ // the orphaning decision at the least informed possible moment — inside
204
+ // a Y/N, before anyone has looked at the machine — and the non-TTY
205
+ // default made orphaning the norm for every scripted removal.
206
+ //
207
+ // The module is marked ERROR rather than left reading INSTALLED,
208
+ // because the previous behaviour recorded the failure only in a
209
+ // `module_operations` row: `module list` and `system doctor` both
210
+ // showed the module as perfectly healthy while its teardown had failed.
211
+ //
212
+ // The way forward is deliberate: go look, clean up by hand, then
213
+ // re-run with --force, which means "I have remediated; drop the
214
+ // record."
215
+ db.update(modules)
216
+ .set({
217
+ state: 'ERROR',
218
+ errorMessage: `on_uninstall failed: ${reason}`,
219
+ updatedAt: new Date(),
220
+ })
221
+ .where(eq(modules.id, moduleId))
222
+ .run();
223
+
224
+ return {
225
+ success: false,
226
+ error: [
227
+ `Cannot remove '${moduleId}': its on_uninstall hook failed, so cross-module state it registered (port forwards, DNS records, web routes) may still exist.`,
228
+ '',
229
+ ` ${reason}`,
230
+ '',
231
+ `'${moduleId}' is now marked ERROR and has NOT been removed. Inspect what the hook left behind, clean it up, then re-run:`,
232
+ ` celilo module remove ${moduleId} --force`,
233
+ '',
234
+ '--force means "I have remediated this by hand; delete the record anyway".',
235
+ ].join('\n'),
236
+ };
204
237
  }
238
+
239
+ log.warn(`on_uninstall failed: ${reason}`);
240
+ log.info(
241
+ '--force set: continuing removal. Any state the hook left behind is yours to clean up.',
242
+ );
205
243
  }
206
244
  }
207
245
 
@@ -5,11 +5,123 @@
5
5
  import { eq } from 'drizzle-orm';
6
6
  import { getDb } from '../../db/client';
7
7
  import { capabilities, moduleConfigs, modules, secrets } from '../../db/schema';
8
+ import type { ModuleManifest } from '../../manifest/schema';
9
+ import {
10
+ HEALTH_CHECK_INTERVAL_CONFIG_KEY,
11
+ effectiveHealthCheckCadence,
12
+ } from '../../services/alerting/health-cadence';
13
+ import {
14
+ BACKUP_RETENTION_COUNT_CONFIG_KEY,
15
+ BACKUP_RETENTION_MAX_AGE_DAYS_CONFIG_KEY,
16
+ effectiveBackupRetention,
17
+ prunesNothing,
18
+ } from '../../services/backup-retention';
19
+ import {
20
+ BACKUP_SCHEDULE_CONFIG_KEY,
21
+ effectiveBackupSchedule,
22
+ } from '../../services/backup-schedule';
23
+ import { formatCadence } from '../../services/cadence';
8
24
  import { getModuleSystems } from '../../services/deployed-systems';
25
+ import { configOverride, parseStoredConfigValue } from '../../services/module-config';
9
26
  import { formatPlacementLine, reconcilePlacement } from '../../services/placement-reconcile';
10
27
  import { getArg, validateRequiredArgs } from '../parser';
11
28
  import type { CommandResult } from '../types';
12
29
 
30
+ /**
31
+ * PURE (Rule 10.1): the per-module policy block — what celilo will do to this
32
+ * module, and on whose authority.
33
+ *
34
+ * Both halves are shown deliberately. The stored override alone does not tell
35
+ * an operator what they changed it FROM, and the effective value alone does not
36
+ * tell them whether they set it or the module's author did.
37
+ */
38
+ export function formatCadencePolicy(input: {
39
+ manifest: ModuleManifest;
40
+ configs: Record<string, unknown>;
41
+ }): string {
42
+ const { manifest, configs } = input;
43
+ const lines = ['Policy:'];
44
+
45
+ if (manifest.hooks?.on_backup) {
46
+ const override = configOverride(configs, BACKUP_SCHEDULE_CONFIG_KEY);
47
+ const effective = formatCadence(effectiveBackupSchedule(manifest, override));
48
+ const suggested = manifest.backup?.schedule;
49
+ if (override !== undefined) {
50
+ lines.push(
51
+ ` backup cadence: ${effective} (operator override; manifest suggests ${suggested ?? 'nothing'})`,
52
+ );
53
+ } else if (suggested !== undefined) {
54
+ lines.push(` backup cadence: ${effective} (from the manifest)`);
55
+ } else {
56
+ lines.push(` backup cadence: ${effective} (celilo default; nothing declared or set)`);
57
+ }
58
+ lines.push(` backup retention: ${describeRetention(manifest, configs)}`);
59
+ } else {
60
+ lines.push(' backup cadence: not backed up (module declares no on_backup hook)');
61
+ }
62
+
63
+ if (manifest.hooks?.health_check) {
64
+ const override = configOverride(configs, HEALTH_CHECK_INTERVAL_CONFIG_KEY);
65
+ const effective = effectiveHealthCheckCadence(manifest, override);
66
+ const suggested = manifest.hooks.health_check.interval;
67
+ const value = effective === null ? 'not watched (no cadence set)' : formatCadence(effective);
68
+ if (override !== undefined) {
69
+ lines.push(
70
+ ` health check: ${value} (operator override; manifest suggests ${suggested ?? 'nothing'})`,
71
+ );
72
+ } else if (suggested !== undefined) {
73
+ lines.push(` health check: ${value} (from the manifest)`);
74
+ } else {
75
+ lines.push(` health check: ${value}`);
76
+ }
77
+ } else {
78
+ lines.push(' health check: not watched (module declares no health_check hook)');
79
+ }
80
+
81
+ return lines.join('\n');
82
+ }
83
+
84
+ /**
85
+ * Retention in one line, per dimension, saying "unbounded" rather than a number
86
+ * wherever nothing bounds it. An operator reading a bound they never set would
87
+ * reasonably assume backups are being deleted — and the reverse assumption,
88
+ * that something is pruning when nothing is, is how a disk fills.
89
+ */
90
+ function describeRetention(manifest: ModuleManifest, configs: Record<string, unknown>): string {
91
+ const policy = effectiveBackupRetention(manifest, configs);
92
+ if (prunesNothing(policy)) return 'none — every backup is kept';
93
+
94
+ const declared = manifest.backup?.retention;
95
+ const dimension = (
96
+ effective: number,
97
+ override: string | undefined,
98
+ suggested: number | undefined,
99
+ unit: string,
100
+ ): string => {
101
+ if (effective === Number.POSITIVE_INFINITY) return `unbounded ${unit}`;
102
+ const source =
103
+ override !== undefined
104
+ ? `operator override; manifest suggests ${suggested ?? 'nothing'}`
105
+ : 'from the manifest';
106
+ return `${effective} ${unit} (${source})`;
107
+ };
108
+
109
+ return [
110
+ dimension(
111
+ policy.count,
112
+ configOverride(configs, BACKUP_RETENTION_COUNT_CONFIG_KEY),
113
+ declared?.count,
114
+ 'copies',
115
+ ),
116
+ dimension(
117
+ policy.maxAgeDays,
118
+ configOverride(configs, BACKUP_RETENTION_MAX_AGE_DAYS_CONFIG_KEY),
119
+ declared?.max_age_days,
120
+ 'days',
121
+ ),
122
+ ].join(', ');
123
+ }
124
+
13
125
  /**
14
126
  * Handle module status command
15
127
  *
@@ -62,10 +174,9 @@ export async function handleModuleStatus(args: string[]): Promise<CommandResult>
62
174
  .where(eq(capabilities.moduleId, moduleId))
63
175
  .all();
64
176
 
65
- // Build sections as multi-line blocks joined by \n\n.
66
- // The display code in index.ts splits on \n\n and passes each section
67
- // as a single multi-line string to p.log.message(), avoiding clack's
68
- // per-line extra spacing.
177
+ // Build sections as multi-line blocks joined by \n\n. index.ts writes the
178
+ // whole message to stdout verbatim, so the blank line between sections is
179
+ // exactly what the operator sees.
69
180
  const sections: string[] = [];
70
181
 
71
182
  // Section 1: Module metadata
@@ -115,6 +226,18 @@ export async function handleModuleStatus(args: string[]): Promise<CommandResult>
115
226
  sections.push('Configuration: (none)');
116
227
  }
117
228
 
229
+ // Section 2b: Per-module policy — what celilo will DO to this module, and
230
+ // whether that came from the operator or from the module's author. A raw
231
+ // config key does not tell an operator what the manifest said, and an
232
+ // effective value alone does not tell them whether they are the one who set
233
+ // it. Both, always.
234
+ sections.push(
235
+ formatCadencePolicy({
236
+ manifest: module.manifestData as ModuleManifest,
237
+ configs: Object.fromEntries(configs.map((c) => [c.key, parseStoredConfigValue(c)])),
238
+ }),
239
+ );
240
+
118
241
  // Section 3: Secrets
119
242
  if (moduleSecrets.length > 0) {
120
243
  const secretLines = ['Secrets:'];
@@ -143,7 +143,7 @@ description: fixture
143
143
  });
144
144
 
145
145
  test('quiet=true suppresses the per-call log lines (caller renders its own)', async () => {
146
- // We can't easily intercept @clack's log output without adding test
146
+ // We can't easily intercept the log helpers' output without adding test
147
147
  // hooks, so instead we exercise that the call simply succeeds and
148
148
  // returns the structured outcome — the sweep relies on this to
149
149
  // render its own output without duplicates. A non-quiet call