@celilo/cli 0.22.0 → 0.24.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (104) hide show
  1. package/CELILO_CORE_MODULES.md +2 -2
  2. package/CELILO_SUBSYSTEMS.md +61 -9
  3. package/drizzle/0024_module_pause.sql +20 -0
  4. package/drizzle/meta/_journal.json +8 -1
  5. package/package.json +7 -7
  6. package/src/__integration__/container-services-cli.integration.test.ts +8 -2
  7. package/src/api/remote-client.test.ts +6 -5
  8. package/src/api/serve.ts +41 -7
  9. package/src/api-clients/proxmox.ts +34 -0
  10. package/src/cli/commands/alerts-act.ts +1 -1
  11. package/src/cli/commands/alerts-sweep.ts +2 -0
  12. package/src/cli/commands/backup-create.ts +26 -11
  13. package/src/cli/commands/backup-list.test.ts +83 -0
  14. package/src/cli/commands/backup-list.ts +67 -3
  15. package/src/cli/commands/backup-prune.ts +17 -17
  16. package/src/cli/commands/backup-sweep.ts +20 -8
  17. package/src/cli/commands/events.ts +34 -3
  18. package/src/cli/commands/firewall-interface-list.test.ts +85 -0
  19. package/src/cli/commands/firewall-interface-list.ts +123 -0
  20. package/src/cli/commands/machine-add.ts +30 -2
  21. package/src/cli/commands/module-config.test.ts +64 -2
  22. package/src/cli/commands/module-config.ts +159 -8
  23. package/src/cli/commands/module-deploy.ts +2 -2
  24. package/src/cli/commands/module-health.ts +1 -0
  25. package/src/cli/commands/module-import.ts +3 -3
  26. package/src/cli/commands/module-list.ts +12 -1
  27. package/src/cli/commands/module-pause.ts +317 -0
  28. package/src/cli/commands/module-remove.ts +78 -40
  29. package/src/cli/commands/module-status.ts +127 -4
  30. package/src/cli/commands/module-update.test.ts +1 -1
  31. package/src/cli/commands/monitor.ts +116 -19
  32. package/src/cli/commands/proxmox-template-selection.ts +1 -1
  33. package/src/cli/commands/status.ts +25 -3
  34. package/src/cli/commands/system-migrate.ts +14 -0
  35. package/src/cli/commands/system-update.ts +4 -1
  36. package/src/cli/completion.ts +39 -9
  37. package/src/cli/fuel-gauge.ts +4 -4
  38. package/src/cli/index.ts +104 -22
  39. package/src/cli/json-output.test.ts +162 -0
  40. package/src/cli/prompts.ts +53 -74
  41. package/src/cli/service-credential.ts +3 -3
  42. package/src/cli/stdout-is-undecorated.test.ts +94 -0
  43. package/src/cli/tui/audit-state.ts +2 -0
  44. package/src/cli/types.ts +7 -2
  45. package/src/db/schema.ts +73 -15
  46. package/src/hooks/capability-loader.ts +130 -4
  47. package/src/hooks/run-named-hook.ts +28 -0
  48. package/src/hooks/types.ts +2 -1
  49. package/src/manifest/contracts/v1.ts +16 -0
  50. package/src/manifest/schema.ts +40 -65
  51. package/src/services/alerting/builtin-monitors.test.ts +18 -10
  52. package/src/services/alerting/cadence-migration.test.ts +155 -0
  53. package/src/services/alerting/cadence-migration.ts +90 -0
  54. package/src/services/alerting/coverage-source.ts +8 -11
  55. package/src/services/alerting/deploy-hooks.test.ts +16 -7
  56. package/src/services/alerting/deploy-hooks.ts +11 -5
  57. package/src/services/alerting/health-cadence.test.ts +58 -0
  58. package/src/services/alerting/health-cadence.ts +128 -0
  59. package/src/services/alerting/health-coverage.ts +18 -8
  60. package/src/services/alerting/monitors.ts +50 -15
  61. package/src/services/alerting/suppression.test.ts +5 -0
  62. package/src/services/alerting/suppression.ts +18 -1
  63. package/src/services/alerting/sweep-runner.test.ts +52 -3
  64. package/src/services/alerting/sweep-runner.ts +41 -8
  65. package/src/services/audit/backup-source.ts +24 -1
  66. package/src/services/audit/backups.test.ts +95 -10
  67. package/src/services/audit/backups.ts +40 -37
  68. package/src/services/audit/interface-classification.test.ts +220 -0
  69. package/src/services/audit/interface-classification.ts +167 -0
  70. package/src/services/audit/types.ts +2 -1
  71. package/src/services/backup-age-agreement.test.ts +118 -0
  72. package/src/services/backup-create.ts +36 -30
  73. package/src/services/backup-metadata.ts +52 -1
  74. package/src/services/backup-retention.test.ts +123 -0
  75. package/src/services/backup-retention.ts +66 -5
  76. package/src/services/backup-schedule.test.ts +166 -0
  77. package/src/services/backup-schedule.ts +105 -15
  78. package/src/services/backup-staging.ts +14 -1
  79. package/src/services/backup-sweep.test.ts +22 -3
  80. package/src/services/backup-sweep.ts +15 -5
  81. package/src/services/bus-interview.ts +2 -2
  82. package/src/services/bus-secret-flow.test.ts +1 -1
  83. package/src/services/cadence.test.ts +97 -0
  84. package/src/services/cadence.ts +165 -0
  85. package/src/services/fleet-checks.ts +48 -0
  86. package/src/services/machine-detector.ts +23 -1
  87. package/src/services/module-config.ts +33 -0
  88. package/src/services/module-deploy.ts +1 -1
  89. package/src/services/module-pause-observability.test.ts +224 -0
  90. package/src/services/module-pause-quiescence.test.ts +163 -0
  91. package/src/services/module-pause.test.ts +573 -0
  92. package/src/services/module-pause.ts +544 -0
  93. package/src/services/remove-guard.test.ts +175 -0
  94. package/src/services/remove-guard.ts +109 -0
  95. package/src/services/storage-providers/s3.test.ts +96 -13
  96. package/src/services/storage-providers/s3.ts +48 -15
  97. package/src/services/terminal-responder.ts +16 -16
  98. package/src/services/update/dep-graph.test.ts +33 -4
  99. package/src/services/update/dep-graph.ts +39 -17
  100. package/src/services/zone-detector.test.ts +34 -3
  101. package/src/services/zone-detector.ts +32 -49
  102. package/src/test-utils/cli.ts +15 -14
  103. package/src/test-utils/integration-guard.ts +26 -0
  104. package/src/test-utils/setup-test-db.ts +13 -23
@@ -14,7 +14,9 @@ import { modules } from '../../db/schema';
14
14
  import { resolveDeployPosture } from '../../services/deploy-posture';
15
15
  import {
16
16
  FRAMEWORK_CONFIG_KEYS,
17
+ handleModuleConfigGet,
17
18
  handleModuleConfigSet,
19
+ handleModuleConfigUnset,
18
20
  validateFrameworkConfigValue,
19
21
  } from './module-config';
20
22
  import { pickUpgradePolicy } from './module-upgrade';
@@ -112,6 +114,30 @@ describe('handleModuleConfigSet — infra-key contract (ISS-0069)', () => {
112
114
  expect(result.success).toBe(false);
113
115
  });
114
116
 
117
+ test('a per-module policy key is settable on a module whose manifest never mentions it', async () => {
118
+ expect((await handleModuleConfigSet(['testmod', 'backup_schedule', '6h'])).success).toBe(true);
119
+ const read = await handleModuleConfigGet(['testmod', 'backup_schedule']);
120
+ expect(read.success).toBe(true);
121
+ if (read.success) expect(read.message).toContain('6h');
122
+ });
123
+
124
+ test('unset removes the override so the module follows its manifest again', async () => {
125
+ await handleModuleConfigSet(['testmod', 'backup_schedule', 'weekly']);
126
+
127
+ const unset = await handleModuleConfigUnset(['testmod', 'backup_schedule']);
128
+ expect(unset.success).toBe(true);
129
+
130
+ expect((await handleModuleConfigGet(['testmod', 'backup_schedule'])).success).toBe(false);
131
+ });
132
+
133
+ test('unsetting a key that was never set reports it and still succeeds', async () => {
134
+ // `unset` states a desired end state. Failing on an already-clean one makes
135
+ // it unusable from any script that cannot check first.
136
+ const result = await handleModuleConfigUnset(['testmod', 'backup_schedule']);
137
+ expect(result.success).toBe(true);
138
+ if (result.success) expect(result.message).toContain('No override set');
139
+ });
140
+
115
141
  test('the valid-keys hint advertises the celilo-managed keys', async () => {
116
142
  const result = await handleModuleConfigSet(['testmod', 'nope', 'x']);
117
143
  expect(result.success).toBe(false);
@@ -127,8 +153,19 @@ describe('validateFrameworkConfigValue (pure)', () => {
127
153
  expect(validateFrameworkConfigValue('app_port', 'anything')).toBeNull();
128
154
  });
129
155
 
130
- test('accepts every documented value', () => {
131
- for (const [key, values] of Object.entries(FRAMEWORK_CONFIG_KEYS)) {
156
+ test('every framework key accepts at least its documented values', () => {
157
+ const documented: Record<string, string[]> = {
158
+ auto_upgrade: ['true', 'false'],
159
+ upgrade_policy: ['by-semver', 'always-safe', 'always-fast'],
160
+ backup_schedule: ['hourly', 'daily', 'weekly', 'monthly', 'manual', '6h'],
161
+ health_check_interval: ['15m', '1h', 'daily', 'manual'],
162
+ backup_retention_count: ['1', '3', '30'],
163
+ backup_retention_max_age_days: ['1', '30', '365'],
164
+ };
165
+ // Every key must be covered, so adding one without deciding what it accepts
166
+ // fails here rather than shipping unvalidated.
167
+ expect(Object.keys(documented).sort()).toEqual(Object.keys(FRAMEWORK_CONFIG_KEYS).sort());
168
+ for (const [key, values] of Object.entries(documented)) {
132
169
  for (const v of values) expect(validateFrameworkConfigValue(key, v)).toBeNull();
133
170
  }
134
171
  });
@@ -136,6 +173,31 @@ describe('validateFrameworkConfigValue (pure)', () => {
136
173
  test('rejects an unlisted value', () => {
137
174
  expect(validateFrameworkConfigValue('upgrade_policy', 'always_safe')).toContain('Allowed');
138
175
  });
176
+
177
+ test('rejects a cadence the sweep that would serve it cannot run', () => {
178
+ // The backup sweep rides an hourly tick; accepting `5m` would leave the
179
+ // operator believing they configured something that can never happen.
180
+ expect(validateFrameworkConfigValue('backup_schedule', '5m')).toContain('hourly');
181
+ // The alerting sweep ticks every five minutes, so the same value is fine there.
182
+ expect(validateFrameworkConfigValue('health_check_interval', '5m')).toBeNull();
183
+ expect(validateFrameworkConfigValue('health_check_interval', '1m')).toContain('Allowed');
184
+ });
185
+
186
+ test('a retention of zero or below is refused rather than read as "keep nothing"', () => {
187
+ // A `0` read as a bound would delete every backup the module has.
188
+ for (const bad of ['0', '-1', '2.5', 'lots']) {
189
+ expect(validateFrameworkConfigValue('backup_retention_count', bad)).toContain('1 or greater');
190
+ expect(validateFrameworkConfigValue('backup_retention_max_age_days', bad)).toContain(
191
+ '1 or greater',
192
+ );
193
+ }
194
+ });
195
+
196
+ test('a misspelled cadence is refused, and says it was not coerced', () => {
197
+ const error = validateFrameworkConfigValue('backup_schedule', 'dailyy');
198
+ expect(error).toContain('named period');
199
+ expect(error).toContain('Rejected rather than coerced');
200
+ });
139
201
  });
140
202
 
141
203
  // The point of the whole control: with always-safe set, a PATCH upgrade — which
@@ -3,9 +3,25 @@
3
3
  */
4
4
 
5
5
  import { eq } from 'drizzle-orm';
6
+ import { z } from 'zod';
6
7
  import { getDb } from '../../db/client';
7
8
  import { modules } from '../../db/schema';
8
9
  import {
10
+ HEALTH_CHECK_INTERVAL_CONFIG_KEY,
11
+ reconcileModuleWatchState,
12
+ } from '../../services/alerting/health-cadence';
13
+ import {
14
+ BACKUP_RETENTION_COUNT_CONFIG_KEY,
15
+ BACKUP_RETENTION_MAX_AGE_DAYS_CONFIG_KEY,
16
+ } from '../../services/backup-retention';
17
+ import { BACKUP_SCHEDULE_CONFIG_KEY } from '../../services/backup-schedule';
18
+ import {
19
+ BACKUP_CADENCE_FLOOR_MINUTES,
20
+ MONITOR_INTERVAL_FLOOR_MINUTES,
21
+ cadenceSchema,
22
+ } from '../../services/cadence';
23
+ import {
24
+ deleteModuleConfig,
9
25
  formatConfigValue,
10
26
  getAllModuleConfigValues,
11
27
  getModuleConfigValue,
@@ -18,6 +34,12 @@ import type { CommandResult } from '../types';
18
34
  * Operator keys that EVERY module accepts, whether or not its manifest declares
19
35
  * them, with their permitted values.
20
36
  *
37
+ * This is also where a module's per-module POLICY lives: how often to back it
38
+ * up, how often to health-check it. A manifest states those as the author's
39
+ * suggestion about a fleet they have never seen; the row an operator writes
40
+ * here wins, and every reader resolves the two at read time so a corrected
41
+ * manifest still reaches installs that have not overridden it.
42
+ *
21
43
  * These describe how celilo TREATS a module (its CD policy), not how the module
22
44
  * configures itself, so gating them on `variables.owns` had it backwards: it
23
45
  * required each module author to opt into being manageable. The failure was
@@ -34,9 +56,69 @@ import type { CommandResult } from '../types';
34
56
  * floor while nothing changed. A safety control that fails open on a typo is
35
57
  * worse than no control.
36
58
  */
37
- export const FRAMEWORK_CONFIG_KEYS: Record<string, readonly string[]> = {
38
- auto_upgrade: ['true', 'false'],
39
- upgrade_policy: ['by-semver', 'always-safe', 'always-fast'],
59
+ export interface FrameworkConfigKey {
60
+ /** Accepts the value the operator typed, or explains what it should be. */
61
+ schema: z.ZodTypeAny;
62
+ /**
63
+ * Why this key is refused rather than coerced. Carried per key rather than
64
+ * derived from the schema's error: the substance is what a WRONG value would
65
+ * silently do, and no validator knows that.
66
+ */
67
+ why: string;
68
+ }
69
+
70
+ const REFUSED_NOT_COERCED_UPGRADE =
71
+ 'Rejected rather than coerced: an unrecognized value silently falls back to the PERMISSIVE default (upgrade_policy → by-semver, which skips the pre-deploy backup on a patch), so a typo would look like it took effect.';
72
+
73
+ const REFUSED_NOT_COERCED_CADENCE =
74
+ 'Rejected rather than coerced: an unrecognized cadence falls back to the manifest\'s suggestion, so a typo would leave the module on the cadence you meant to change — visibly "set", and doing nothing.';
75
+
76
+ const REFUSED_NOT_COERCED_RETENTION =
77
+ 'Rejected rather than coerced: an unrecognized retention value falls back to the manifest, so a typo would leave the module keeping a different number of backups than you asked for — and the direction that goes wrong deletes data.';
78
+
79
+ /** An enum whose rejection message reads as an operator instruction, not a type error. */
80
+ function oneOf(values: readonly [string, ...string[]]): z.ZodTypeAny {
81
+ return z.enum(values, { errorMap: () => ({ message: `Allowed: ${values.join(', ')}` }) });
82
+ }
83
+
84
+ /**
85
+ * A whole number of things, at least one. Zero is refused rather than read as
86
+ * "keep nothing": a retention of 0 would delete every backup the module has.
87
+ */
88
+ function positiveInteger(what: string): z.ZodTypeAny {
89
+ return z.string().superRefine((value, ctx) => {
90
+ const parsed = Number(value);
91
+ if (!Number.isInteger(parsed) || parsed < 1) {
92
+ ctx.addIssue({
93
+ code: z.ZodIssueCode.custom,
94
+ message: `Allowed: a whole number of ${what}, 1 or greater. Unset the key to keep everything.`,
95
+ });
96
+ }
97
+ });
98
+ }
99
+
100
+ export const FRAMEWORK_CONFIG_KEYS: Record<string, FrameworkConfigKey> = {
101
+ auto_upgrade: { schema: oneOf(['true', 'false']), why: REFUSED_NOT_COERCED_UPGRADE },
102
+ upgrade_policy: {
103
+ schema: oneOf(['by-semver', 'always-safe', 'always-fast']),
104
+ why: REFUSED_NOT_COERCED_UPGRADE,
105
+ },
106
+ [BACKUP_SCHEDULE_CONFIG_KEY]: {
107
+ schema: cadenceSchema({ floorMinutes: BACKUP_CADENCE_FLOOR_MINUTES }),
108
+ why: REFUSED_NOT_COERCED_CADENCE,
109
+ },
110
+ [HEALTH_CHECK_INTERVAL_CONFIG_KEY]: {
111
+ schema: cadenceSchema({ floorMinutes: MONITOR_INTERVAL_FLOOR_MINUTES }),
112
+ why: REFUSED_NOT_COERCED_CADENCE,
113
+ },
114
+ [BACKUP_RETENTION_COUNT_CONFIG_KEY]: {
115
+ schema: positiveInteger('copies to keep'),
116
+ why: REFUSED_NOT_COERCED_RETENTION,
117
+ },
118
+ [BACKUP_RETENTION_MAX_AGE_DAYS_CONFIG_KEY]: {
119
+ schema: positiveInteger('days to keep a backup'),
120
+ why: REFUSED_NOT_COERCED_RETENTION,
121
+ },
40
122
  };
41
123
 
42
124
  /**
@@ -44,10 +126,12 @@ export const FRAMEWORK_CONFIG_KEYS: Record<string, readonly string[]> = {
44
126
  * or null when the key is not a framework key or the value is permitted.
45
127
  */
46
128
  export function validateFrameworkConfigValue(key: string, value: string): string | null {
47
- const allowed = FRAMEWORK_CONFIG_KEYS[key];
48
- if (!allowed) return null;
49
- if (allowed.includes(value)) return null;
50
- return `Invalid value '${value}' for '${key}'.\n\nAllowed: ${allowed.join(', ')}\n\nRejected rather than coerced: an unrecognized value silently falls back to the PERMISSIVE default (upgrade_policy → by-semver, which skips the pre-deploy backup on a patch), so a typo would look like it took effect.`;
129
+ const framework = FRAMEWORK_CONFIG_KEYS[key];
130
+ if (!framework) return null;
131
+ const result = framework.schema.safeParse(value);
132
+ if (result.success) return null;
133
+ const explanation = result.error.issues.map((issue) => issue.message).join('\n');
134
+ return `Invalid value '${value}' for '${key}'.\n\n${explanation}\n\n${framework.why}`;
51
135
  }
52
136
 
53
137
  /**
@@ -131,10 +215,14 @@ export async function handleModuleConfigSet(args: string[]): Promise<CommandResu
131
215
  // Set config value using service (handles primitive and complex types)
132
216
  try {
133
217
  await setModuleConfigValue(moduleId, key, value);
218
+ const resolved = settlingWatchState(db, moduleId, key);
134
219
 
135
220
  return {
136
221
  success: true,
137
- message: `Set config for ${moduleId}: ${key}`,
222
+ message:
223
+ resolved > 0
224
+ ? `Set config for ${moduleId}: ${key} (resolved ${resolved} alert(s) — nothing will report on this module again until it is watched)`
225
+ : `Set config for ${moduleId}: ${key}`,
138
226
  };
139
227
  } catch (error) {
140
228
  return {
@@ -144,6 +232,69 @@ export async function handleModuleConfigSet(args: string[]): Promise<CommandResu
144
232
  }
145
233
  }
146
234
 
235
+ /**
236
+ * Handle module config unset command
237
+ *
238
+ * Usage: celilo module config unset <module-id> <key>
239
+ *
240
+ * Removing an override is what returns a module to following its manifest.
241
+ * Without it, an operator who once set a cadence could never go back to the
242
+ * author's suggestion, and every later correction would stop reaching them —
243
+ * the exact failure read-time resolution exists to prevent, aimed at the
244
+ * operators who engaged with the feature.
245
+ *
246
+ * @param args - Command arguments
247
+ * @returns Command result
248
+ */
249
+ export async function handleModuleConfigUnset(args: string[]): Promise<CommandResult> {
250
+ const error = validateRequiredArgs(args, 2);
251
+ if (error) {
252
+ return {
253
+ success: false,
254
+ error: `${error}\n\nUsage: celilo module config unset <module-id> <key>`,
255
+ };
256
+ }
257
+
258
+ const moduleId = getArg(args, 0);
259
+ const key = getArg(args, 1);
260
+
261
+ if (!moduleId || !key) {
262
+ return { success: false, error: 'Module ID and key are required' };
263
+ }
264
+
265
+ const db = getDb();
266
+
267
+ const module = db.select().from(modules).where(eq(modules.id, moduleId)).get();
268
+ if (!module) {
269
+ return { success: false, error: `Module not found: ${moduleId}` };
270
+ }
271
+
272
+ // Unsetting a key that was never set SUCCEEDS. `unset` states a desired end
273
+ // state, and failing on an already-clean one makes it unusable from any
274
+ // script that cannot check first.
275
+ if (!getModuleConfigValue(moduleId, key, db)) {
276
+ return { success: true, message: `No override set for ${moduleId}: ${key}` };
277
+ }
278
+
279
+ deleteModuleConfig(db, moduleId, key);
280
+ settlingWatchState(db, moduleId, key);
281
+ return {
282
+ success: true,
283
+ message: `Unset config for ${moduleId}: ${key} (now follows the module's manifest)`,
284
+ };
285
+ }
286
+
287
+ /**
288
+ * A module that has just stopped being watched may still own live alerts from
289
+ * its last scheduled runs, and nothing will ever report on them again. Resolve
290
+ * them here rather than leaving them firing with no action able to clear them.
291
+ * No-op for every other key.
292
+ */
293
+ function settlingWatchState(db: ReturnType<typeof getDb>, moduleId: string, key: string): number {
294
+ if (key !== HEALTH_CHECK_INTERVAL_CONFIG_KEY) return 0;
295
+ return reconcileModuleWatchState(db, moduleId, new Date());
296
+ }
297
+
147
298
  /**
148
299
  * Handle module config get command
149
300
  *
@@ -71,8 +71,8 @@ export async function handleModuleDeploy(
71
71
  }
72
72
 
73
73
  // Success message is emitted by deployModule via the active ProgressDisplay,
74
- // so we return an empty message to avoid the top-level clack outro printing
75
- // a duplicate line in a different style.
74
+ // so we return an empty message index.ts exits without writing anything
75
+ // more, rather than printing a duplicate line in a different style.
76
76
  return {
77
77
  success: true,
78
78
  message: '',
@@ -79,6 +79,7 @@ export async function handleModuleHealth(
79
79
  return {
80
80
  success: true,
81
81
  message: JSON.stringify({ modules: results }, null, 2),
82
+ rawOutput: true,
82
83
  };
83
84
  }
84
85
 
@@ -143,9 +143,9 @@ export async function handleAspectApprovalAfterImport(args: {
143
143
  `celilo will run the '${aspect.ansible_role}' Ansible role on every non-api_only system`,
144
144
  'in those zones.',
145
145
  ].join('\n');
146
- // Use process.stderr.write rather than the prompt UI so the
147
- // multi-line scope block is preserved verbatim — @clack/prompts
148
- // mangles multi-line message arguments.
146
+ // Written straight to stderr rather than through the prompt UI so the
147
+ // multi-line scope block is preserved verbatim — a prompt renderer owns
148
+ // its frame and reflows a multi-line message argument.
149
149
  process.stderr.write(`\n${scopeMsg}\n\n`);
150
150
 
151
151
  const accepted = await withInterviewSession(() =>
@@ -3,6 +3,7 @@ import { type DbClient, getDb } from '../../db/client';
3
3
  import { modules, modules as modulesTable, monitors } from '../../db/schema';
4
4
  import { moduleHealthCell } from '../../services/alerting/format';
5
5
  import { loadAllLiveAlerts, summariseByModule } from '../../services/alerting/store';
6
+ import { formatPausedDuration } from '../../services/module-pause';
6
7
  import { hasFlag } from '../parser';
7
8
  import type { CommandResult } from '../types';
8
9
 
@@ -50,7 +51,17 @@ export async function handleModuleList(
50
51
  for (const module of moduleRows) {
51
52
  const observed = health.get(module.id);
52
53
  const healthNote = observed ? ` [${observed}]` : '';
53
- lines.push(`${module.id} (v${module.version}) - ${module.state}${healthNote}`);
54
+ // A pause suppresses the alerting that would otherwise report this module
55
+ // as down, so the state alone is not enough — the AGE is what separates a
56
+ // maintenance window from an outage nobody remembers taking (design D7).
57
+ const stateCell =
58
+ module.state === 'PAUSED'
59
+ ? `PAUSED (${formatPausedDuration(module.pausedAt)})`
60
+ : module.state;
61
+ lines.push(`${module.id} (v${module.version}) - ${stateCell}${healthNote}`);
62
+ if (module.state === 'PAUSED' && module.pauseReason) {
63
+ lines.push(` Paused: ${module.pauseReason}`);
64
+ }
54
65
  if (module.description) {
55
66
  lines.push(` ${module.description}`);
56
67
  }