@celilo/cli 0.22.0 → 0.23.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (51) hide show
  1. package/CELILO_SUBSYSTEMS.md +34 -2
  2. package/drizzle/0024_module_pause.sql +20 -0
  3. package/drizzle/meta/_journal.json +8 -1
  4. package/package.json +4 -5
  5. package/src/__integration__/container-services-cli.integration.test.ts +8 -2
  6. package/src/api/remote-client.test.ts +6 -5
  7. package/src/api/serve.ts +41 -7
  8. package/src/api-clients/proxmox.ts +34 -0
  9. package/src/cli/commands/alerts-sweep.ts +2 -0
  10. package/src/cli/commands/events.ts +34 -3
  11. package/src/cli/commands/module-deploy.ts +2 -2
  12. package/src/cli/commands/module-health.ts +1 -0
  13. package/src/cli/commands/module-import.ts +3 -3
  14. package/src/cli/commands/module-list.ts +12 -1
  15. package/src/cli/commands/module-pause.ts +317 -0
  16. package/src/cli/commands/module-remove.ts +78 -40
  17. package/src/cli/commands/module-status.ts +3 -4
  18. package/src/cli/commands/module-update.test.ts +1 -1
  19. package/src/cli/commands/proxmox-template-selection.ts +1 -1
  20. package/src/cli/commands/status.ts +25 -3
  21. package/src/cli/completion.ts +4 -0
  22. package/src/cli/fuel-gauge.ts +4 -4
  23. package/src/cli/index.ts +45 -20
  24. package/src/cli/json-output.test.ts +162 -0
  25. package/src/cli/prompts.ts +53 -74
  26. package/src/cli/service-credential.ts +3 -3
  27. package/src/cli/stdout-is-undecorated.test.ts +94 -0
  28. package/src/cli/types.ts +7 -2
  29. package/src/db/schema.ts +73 -15
  30. package/src/hooks/run-named-hook.ts +28 -0
  31. package/src/services/alerting/suppression.test.ts +5 -0
  32. package/src/services/alerting/suppression.ts +18 -1
  33. package/src/services/alerting/sweep-runner.test.ts +1 -0
  34. package/src/services/alerting/sweep-runner.ts +11 -1
  35. package/src/services/bus-interview.ts +2 -2
  36. package/src/services/bus-secret-flow.test.ts +1 -1
  37. package/src/services/fleet-checks.ts +48 -0
  38. package/src/services/module-deploy.ts +1 -1
  39. package/src/services/module-pause-observability.test.ts +224 -0
  40. package/src/services/module-pause-quiescence.test.ts +163 -0
  41. package/src/services/module-pause.test.ts +573 -0
  42. package/src/services/module-pause.ts +544 -0
  43. package/src/services/remove-guard.test.ts +175 -0
  44. package/src/services/remove-guard.ts +109 -0
  45. package/src/services/terminal-responder.ts +16 -16
  46. package/src/services/update/dep-graph.test.ts +33 -4
  47. package/src/services/update/dep-graph.ts +39 -17
  48. package/src/services/zone-detector.ts +2 -39
  49. package/src/test-utils/cli.ts +15 -14
  50. package/src/test-utils/integration-guard.ts +26 -0
  51. package/src/test-utils/setup-test-db.ts +13 -23
package/src/db/schema.ts CHANGED
@@ -11,7 +11,17 @@ import {
11
11
 
12
12
  /**
13
13
  * Module lifecycle states
14
- * IMPORTED, VALIDATED, CONFIGURED, GENERATING, ERROR, DEPLOYING, INSTALLED, VERIFIED, UNINSTALLING
14
+ * IMPORTED, VALIDATED, CONFIGURED, GENERATING, ERROR, DEPLOYING, INSTALLED, VERIFIED, UNINSTALLING, PAUSED
15
+ *
16
+ * `PAUSED` is a real member of this union rather than a side flag, and that is
17
+ * the point (openspec/changes/module-pause-lifecycle/design.md D1): adding it
18
+ * makes the type-checker enumerate every site that must now consider
19
+ * paused-ness. A `pausedAt`-only flag would leave every `state === 'VERIFIED'`
20
+ * comparison silently compiling while quietly reading a paused module as live.
21
+ *
22
+ * There is deliberately no `prePauseState`: pause is legal only from a settled
23
+ * state and unpause redeploys, so the deploy path decides the resulting state
24
+ * and there is nothing to restore.
15
25
  */
16
26
  export type ModuleState =
17
27
  | 'IMPORTED'
@@ -22,23 +32,65 @@ export type ModuleState =
22
32
  | 'INSTALLED'
23
33
  | 'VERIFIED'
24
34
  | 'ERROR'
25
- | 'UNINSTALLING';
35
+ | 'UNINSTALLING'
36
+ | 'PAUSED';
37
+
38
+ /**
39
+ * States a module may be paused FROM (design D1). `ERROR` is deliberately
40
+ * included: quiescing a broken module to stop alert noise while working on it
41
+ * is legitimate, and refusing would push the operator toward silencing those
42
+ * alerts by some less visible route.
43
+ */
44
+ export const PAUSABLE_STATES = [
45
+ 'INSTALLED',
46
+ 'VERIFIED',
47
+ 'ERROR',
48
+ ] as const satisfies readonly ModuleState[];
49
+
50
+ /**
51
+ * States that mean "a lifecycle transition is under way". Pausing one of these
52
+ * would strand the transition, so pause is refused with a distinct message from
53
+ * the never-deployed case.
54
+ */
55
+ export const IN_FLIGHT_STATES = [
56
+ 'GENERATING',
57
+ 'DEPLOYING',
58
+ 'UNINSTALLING',
59
+ ] as const satisfies readonly ModuleState[];
26
60
 
27
61
  /**
28
62
  * Modules table - stores module metadata and manifest data
29
63
  */
30
- export const modules = sqliteTable('modules', {
31
- id: text('id').primaryKey(),
32
- name: text('name').notNull(),
33
- version: text('version').notNull(),
34
- description: text('description'),
35
- state: text('state').$type<ModuleState>().notNull().default('IMPORTED'),
36
- manifestData: text('manifest_data', { mode: 'json' }).$type<Record<string, unknown>>().notNull(),
37
- sourcePath: text('source_path').notNull(),
38
- importedAt: integer('imported_at', { mode: 'timestamp' }).notNull().default(sql`(unixepoch())`),
39
- updatedAt: integer('updated_at', { mode: 'timestamp' }).notNull().default(sql`(unixepoch())`),
40
- errorMessage: text('error_message'),
41
- });
64
+ export const modules = sqliteTable(
65
+ 'modules',
66
+ {
67
+ id: text('id').primaryKey(),
68
+ name: text('name').notNull(),
69
+ version: text('version').notNull(),
70
+ description: text('description'),
71
+ state: text('state').$type<ModuleState>().notNull().default('IMPORTED'),
72
+ manifestData: text('manifest_data', { mode: 'json' })
73
+ .$type<Record<string, unknown>>()
74
+ .notNull(),
75
+ sourcePath: text('source_path').notNull(),
76
+ importedAt: integer('imported_at', { mode: 'timestamp' }).notNull().default(sql`(unixepoch())`),
77
+ updatedAt: integer('updated_at', { mode: 'timestamp' }).notNull().default(sql`(unixepoch())`),
78
+ errorMessage: text('error_message'),
79
+ /**
80
+ * When the module was paused. Null unless `state = 'PAUSED'`. The state alone
81
+ * cannot answer "how long", and the DURATION is what makes a forgotten pause
82
+ * detectable (design D7) — every paused-module report carries the age.
83
+ */
84
+ pausedAt: integer('paused_at', { mode: 'timestamp' }),
85
+ /** Operator-supplied explanation, so the row explains itself. */
86
+ pauseReason: text('pause_reason'),
87
+ },
88
+ (table) => ({
89
+ // Every management-API response asks "is anything paused?" (design D7), so
90
+ // that lookup must stay a single indexed hit rather than a table scan.
91
+ stateIdx: index('modules_state_idx').on(table.state),
92
+ }),
93
+ );
42
94
 
43
95
  /**
44
96
  * Module configuration - user-provided key-value pairs.
@@ -807,7 +859,13 @@ export const backups = sqliteTable('backups', {
807
859
  * pid is no longer alive is treated as abandoned (the process crashed before
808
860
  * the completion update landed) and ignored by in-flight checks.
809
861
  */
810
- export type ModuleOperationKind = 'deploy' | 'uninstall' | 'backup' | 'restore';
862
+ export type ModuleOperationKind =
863
+ | 'deploy'
864
+ | 'uninstall'
865
+ | 'backup'
866
+ | 'restore'
867
+ | 'pause'
868
+ | 'unpause';
811
869
  export type ModuleOperationStatus = 'in_progress' | 'completed' | 'failed';
812
870
 
813
871
  export const moduleOperations = sqliteTable('module_operations', {
@@ -49,6 +49,13 @@ export interface RunNamedHookResult extends HookResult {
49
49
  * gracefully when a module has no `on_uninstall` defined.
50
50
  */
51
51
  notDefined?: boolean;
52
+ /**
53
+ * True when the hook was not run because the module is PAUSED. Reported as
54
+ * success rather than failure: the module is deliberately quiesced, and a
55
+ * failure here would be retried by the bus and then alerted on — paging the
56
+ * operator about the pause they took themselves.
57
+ */
58
+ skippedPaused?: boolean;
52
59
  }
53
60
 
54
61
  /**
@@ -82,6 +89,27 @@ export async function runNamedHook(
82
89
  };
83
90
  }
84
91
 
92
+ // Quiescence for a paused module (openspec/changes/module-pause-lifecycle,
93
+ // tasks 2.1/2.2). This is the chokepoint every non-lifecycle invocation
94
+ // funnels through — bus dispatch, timer fan-out, aspect fan-out,
95
+ // public-web republish, the dns-provider backfill, and `module run-hook` —
96
+ // so guarding here covers the paths individually rather than each caller
97
+ // remembering to.
98
+ //
99
+ // The exemptions are decided from the hook NAME, not a caller-supplied flag
100
+ // (Rule 10.3): `on_install` is how unpause redeploys the module back to life,
101
+ // and `on_uninstall` is how a paused module is removed — which is the entire
102
+ // point of pausing it. Both must run while `state` is still PAUSED.
103
+ const LIFECYCLE_HOOKS: readonly HookName[] = ['on_install', 'on_uninstall'];
104
+ if (module.state === 'PAUSED' && !LIFECYCLE_HOOKS.includes(hookName)) {
105
+ return {
106
+ success: true,
107
+ outputs: {},
108
+ duration: Date.now() - startedAt,
109
+ skippedPaused: true,
110
+ };
111
+ }
112
+
85
113
  const manifest = module.manifestData as ModuleManifest;
86
114
  const hookDef = manifest.hooks?.[hookName as keyof typeof manifest.hooks];
87
115
  if (!hookDef) {
@@ -35,6 +35,7 @@ const TOPOLOGY: SuppressionTopology = {
35
35
  const noSuppression = {
36
36
  suppressible: true,
37
37
  modulesInDeployWindow: new Set<string>(),
38
+ pausedModules: new Set<string>(),
38
39
  topology: TOPOLOGY,
39
40
  };
40
41
 
@@ -185,6 +186,7 @@ describe('findSuppressor — guards', () => {
185
186
  firingKeys,
186
187
  suppressible: false,
187
188
  modulesInDeployWindow: new Set(),
189
+ pausedModules: new Set(),
188
190
  topology: TOPOLOGY,
189
191
  }),
190
192
  ).toBeNull();
@@ -197,6 +199,7 @@ describe('findSuppressor — guards', () => {
197
199
  firingKeys: new Set(),
198
200
  suppressible: true,
199
201
  modulesInDeployWindow: new Set(['forgejo']),
202
+ pausedModules: new Set(),
200
203
  topology: TOPOLOGY,
201
204
  }),
202
205
  ).toEqual({ kind: 'deploy_window', moduleId: 'forgejo' });
@@ -209,6 +212,7 @@ describe('findSuppressor — guards', () => {
209
212
  firingKeys: new Set(),
210
213
  suppressible: true,
211
214
  modulesInDeployWindow: new Set(['forgejo']),
215
+ pausedModules: new Set(),
212
216
  topology: TOPOLOGY,
213
217
  }),
214
218
  ).toBeNull();
@@ -221,6 +225,7 @@ describe('findSuppressor — guards', () => {
221
225
  firingKeys: new Set([machineAlertKey('iot')]),
222
226
  suppressible: true,
223
227
  modulesInDeployWindow: new Set(['homebridge']),
228
+ pausedModules: new Set(),
224
229
  topology: TOPOLOGY,
225
230
  }),
226
231
  ).toEqual({ kind: 'deploy_window', moduleId: 'homebridge' });
@@ -106,12 +106,20 @@ export interface SuppressorLookup {
106
106
  suppressible: boolean;
107
107
  /** Modules currently inside a deploy window. */
108
108
  modulesInDeployWindow: ReadonlySet<string>;
109
+ /**
110
+ * Modules currently PAUSED. A pause is a deliberate quiescing, so its alerts
111
+ * are explained by the pause itself (openspec/changes/module-pause-lifecycle,
112
+ * task 2.3) — same mechanism as a deploy window, with the pause as the source
113
+ * instead of an ancestor alert.
114
+ */
115
+ pausedModules: ReadonlySet<string>;
109
116
  topology: SuppressionTopology;
110
117
  }
111
118
 
112
119
  export type Suppressor =
113
120
  | { kind: 'alert'; key: string }
114
- | { kind: 'deploy_window'; moduleId: string };
121
+ | { kind: 'deploy_window'; moduleId: string }
122
+ | { kind: 'paused'; moduleId: string };
115
123
 
116
124
  /**
117
125
  * Find what is suppressing `key`, or null if it should be reported.
@@ -128,6 +136,15 @@ export function findSuppressor(lookup: SuppressorLookup): Suppressor | null {
128
136
 
129
137
  const parsed = parseAlertKey(lookup.key);
130
138
 
139
+ // A pause is checked before a deploy window because it is the longer-lived
140
+ // and more consequential explanation: a paused module may also be inside a
141
+ // deploy window (unpause redeploys), and "paused" is the fact the operator
142
+ // needs to see. Attributed to the pause rather than suppressed anonymously —
143
+ // silently dropping the alert is what turns a pause into an invisible outage.
144
+ if (parsed?.source === 'module' && lookup.pausedModules.has(parsed.moduleId)) {
145
+ return { kind: 'paused', moduleId: parsed.moduleId };
146
+ }
147
+
131
148
  // A deploy is the same mechanism with a window as the source instead of an
132
149
  // ancestor alert — which is why deploy auto-silencing is not a second feature.
133
150
  if (parsed?.source === 'module' && lookup.modulesInDeployWindow.has(parsed.moduleId)) {
@@ -50,6 +50,7 @@ describe('runSweep', () => {
50
50
  monitorDeps: monitorDeps(result, now),
51
51
  loadTopology: () => TOPOLOGY,
52
52
  loadDeployWindowModules: () => new Set(),
53
+ loadPausedModules: () => new Set(),
53
54
  isSuppressible: () => true,
54
55
  // No routes configured: the sweep must still run everything else.
55
56
  notifyDepsFor: () => null,
@@ -43,6 +43,8 @@ export interface SweepDeps {
43
43
  loadTopology(): SuppressionTopology;
44
44
  /** Modules currently inside a deploy window. */
45
45
  loadDeployWindowModules(): Set<string>;
46
+ /** Ids of modules currently PAUSED — a pause explains its own module's alerts. */
47
+ loadPausedModules(): Set<string>;
46
48
  /** Whether the monitor owning an alert may be suppressed at all. */
47
49
  isSuppressible(alert: Alert): boolean;
48
50
  /** Compose the per-alert notification context. Null when nothing can page. */
@@ -156,6 +158,7 @@ export async function runSweep(
156
158
  );
157
159
  const topology = deps.loadTopology();
158
160
  const deployWindows = deps.loadDeployWindowModules();
161
+ const paused = deps.loadPausedModules();
159
162
 
160
163
  for (const alert of live) {
161
164
  const suppressor = findSuppressor({
@@ -163,13 +166,20 @@ export async function runSweep(
163
166
  firingKeys,
164
167
  suppressible: deps.isSuppressible(alert),
165
168
  modulesInDeployWindow: deployWindows,
169
+ pausedModules: paused,
166
170
  topology,
167
171
  });
168
172
  const wasSuppressed = alert.state === 'suppressed';
169
173
  if (suppressor && !wasSuppressed) {
170
174
  markSuppressed(db, alert.id, {
171
175
  alertId: suppressor.kind === 'alert' ? suppressor.key : undefined,
172
- windowId: suppressor.kind === 'deploy_window' ? suppressor.moduleId : undefined,
176
+ // A pause is recorded on the same column as a deploy window: both are
177
+ // "a module-scoped condition explains this", and the operator reads the
178
+ // module id either way.
179
+ windowId:
180
+ suppressor.kind === 'deploy_window' || suppressor.kind === 'paused'
181
+ ? suppressor.moduleId
182
+ : undefined,
173
183
  });
174
184
  report.suppressed++;
175
185
  } else if (!suppressor && wasSuppressed) {
@@ -314,7 +314,7 @@ export async function busInterviewGuarded<TReply>(
314
314
  * Ask a single generic interview question over the bus and return the
315
315
  * responder's answer (ISS-0127). The generic counterpart to the deploy's
316
316
  * config/secret/ensure interview: any operator command can call this to make
317
- * its prompts headlessly drivable instead of calling clack directly.
317
+ * its prompts headlessly drivable instead of prompting on stdin directly.
318
318
  *
319
319
  * The return type is `unknown` because the runtime shape depends on
320
320
  * `payload.kind`; prefer the typed wrappers (`askText`, `askSelect`,
@@ -345,7 +345,7 @@ export async function askInterview(
345
345
  *
346
346
  * This is the shared lifecycle every migrated command wraps its interview in,
347
347
  * so the start/close boilerplate lives in exactly one place. The dynamic
348
- * import keeps clack out of the non-TTY path's module graph.
348
+ * import keeps the terminal renderer out of the non-TTY path's module graph.
349
349
  */
350
350
  export async function withInterviewSession<T>(fn: () => Promise<T>): Promise<T> {
351
351
  const responder = process.stdin.isTTY
@@ -3,7 +3,7 @@
3
3
  *
4
4
  * Drives `interviewForMissingSecrets` against a real sqlite bus + real
5
5
  * encrypted store + a programmatic test responder. No fixture modules,
6
- * no machines, no clack — the responder is just a `bus.watch` that
6
+ * no machines, no terminal — the responder is just a `bus.watch` that
7
7
  * mimics what `terminal-responder.ts` does.
8
8
  *
9
9
  * Covers what stage 3 introduced:
@@ -38,6 +38,7 @@ import {
38
38
  readInstalledUnit,
39
39
  unitMainPid,
40
40
  } from './events-daemon';
41
+ import { describePausedModule, listPausedModules } from './module-pause';
41
42
  import { resolveSubscription } from './module-subscriptions';
42
43
 
43
44
  /**
@@ -854,6 +855,52 @@ export function checkControlPlaneNetwork(db: DbClient): FleetFinding {
854
855
  };
855
856
  }
856
857
 
858
+ /**
859
+ * Any paused module is a doctor failure — no threshold, whatever its age
860
+ * (openspec/changes/module-pause-lifecycle, design D7, closed at review).
861
+ *
862
+ * A pause deliberately switches OFF the alerting that would otherwise report
863
+ * the module as down, so the paused-ness itself has to be the signal. A
864
+ * duration threshold was considered and dropped: there is no number of hours
865
+ * after which a deliberate outage becomes acceptable, and a configurable one is
866
+ * just an invitation to tune the detector until it stops firing.
867
+ *
868
+ * The fleet had just run 20 hours of failing forgejo backups whose only symptom
869
+ * was a column of `0 B` rows that read as healthy hourly cadence. Same failure
870
+ * shape; this is the check that would have named it.
871
+ */
872
+ export function checkPausedModules(db: DbClient): FleetFinding {
873
+ const paused = listPausedModules(db);
874
+
875
+ if (paused.length === 0) {
876
+ return {
877
+ id: 'paused-modules',
878
+ title: 'No module is paused (a pause suppresses its own alerting)',
879
+ status: 'ok',
880
+ summary: 'nothing paused',
881
+ detail: [],
882
+ remediation: null,
883
+ autoFixable: false,
884
+ };
885
+ }
886
+
887
+ const names = paused.map((m) => describePausedModule(m));
888
+ return {
889
+ id: 'paused-modules',
890
+ title: 'No module is paused (a pause suppresses its own alerting)',
891
+ status: 'fail',
892
+ summary: `${paused.length} module(s) paused: ${paused.map((m) => m.id).join(', ')}`,
893
+ detail: [
894
+ ...names.map((n) => `paused: ${n}`),
895
+ 'a paused module receives no dispatched work, runs no health checks, and has its alerts suppressed',
896
+ 'it is still deployed and may still be serving traffic — pause does not stop the data plane',
897
+ ],
898
+ remediation:
899
+ 'bring each back with "celilo module unpause <id>" (which redeploys it, rebinding its capabilities), or remove it if the pause was permanent',
900
+ autoFixable: false,
901
+ };
902
+ }
903
+
857
904
  export async function runFleetChecks(
858
905
  bus: Bus,
859
906
  db: DbClient,
@@ -865,6 +912,7 @@ export async function runFleetChecks(
865
912
  checkSubscribers(bus, db),
866
913
  checkCapabilityProviders(db),
867
914
  checkControlPlaneNetwork(db),
915
+ checkPausedModules(db),
868
916
  await checkServiceDns(db),
869
917
  ];
870
918
  }
@@ -350,7 +350,7 @@ async function deployModuleImpl(
350
350
 
351
351
  // Terminal-responder: when running on a TTY, this subscribes to
352
352
  // `config.required.*` / `secret.required.*` / `ensure.required.*`
353
- // events and prompts via clack. Other responder shapes (Claude
353
+ // events and prompts on the terminal. Other responder shapes (Claude
354
354
  // subagent, `celilo events respond` from another shell, autoresponder
355
355
  // daemon) compete on the bus; first reply wins. See
356
356
  // infra/openspec/changes/interactive-deploys-via-event-bus/proposal.md.
@@ -0,0 +1,224 @@
1
+ /**
2
+ * The parts of pause that exist so a pause cannot become a silent outage
3
+ * (design D7), plus the alerting suppressor.
4
+ *
5
+ * Every check here is asserted in BOTH directions per Rule 7.6 — a gate nobody
6
+ * has seen fail is not a gate, and "no paused module was reported" is exactly
7
+ * the output a broken detector produces.
8
+ */
9
+
10
+ import { describe, expect, test } from 'bun:test';
11
+ import { mkdtempSync } from 'node:fs';
12
+ import { tmpdir } from 'node:os';
13
+ import { join } from 'node:path';
14
+ import { type DbClient, createDbClient } from '../db/client';
15
+ import { type ModuleState, modules } from '../db/schema';
16
+ import { moduleAlertKey } from './alerting/keys';
17
+ import { findSuppressor } from './alerting/suppression';
18
+ import { checkPausedModules } from './fleet-checks';
19
+ import {
20
+ describeMachineStopInfra,
21
+ describePausedModule,
22
+ listPausedModules,
23
+ pausedAmong,
24
+ } from './module-pause';
25
+
26
+ function makeDb(): DbClient {
27
+ const dir = mkdtempSync(join(tmpdir(), 'celilo-pause-obs-'));
28
+ process.env.CELILO_DB_PATH = join(dir, 'celilo.db');
29
+ return createDbClient({ path: process.env.CELILO_DB_PATH });
30
+ }
31
+
32
+ function insert(
33
+ db: DbClient,
34
+ id: string,
35
+ state: ModuleState,
36
+ opts: { pausedAt?: Date; reason?: string } = {},
37
+ ): void {
38
+ db.insert(modules)
39
+ .values({
40
+ id,
41
+ name: id,
42
+ version: '1.0.0',
43
+ state,
44
+ manifestData: { id, name: id, version: '1.0.0' },
45
+ sourcePath: `/tmp/${id}`,
46
+ pausedAt: state === 'PAUSED' ? (opts.pausedAt ?? new Date()) : null,
47
+ pauseReason: state === 'PAUSED' ? (opts.reason ?? null) : null,
48
+ })
49
+ .run();
50
+ }
51
+
52
+ describe('system doctor reports ANY paused module (task 6.2 / 6.4)', () => {
53
+ test('with nothing paused the check passes', () => {
54
+ const db = makeDb();
55
+ insert(db, 'caddy', 'VERIFIED');
56
+
57
+ const finding = checkPausedModules(db);
58
+ expect(finding.status).toBe('ok');
59
+ expect(finding.summary).toBe('nothing paused');
60
+ });
61
+
62
+ test('PROVE IT FAILS: one paused module is a doctor FAILURE naming it and its age', () => {
63
+ const db = makeDb();
64
+ insert(db, 'caddy', 'VERIFIED');
65
+ insert(db, 'greenwave', 'PAUSED', {
66
+ pausedAt: new Date(Date.now() - 3 * 24 * 60 * 60 * 1000),
67
+ reason: 'edge router swap',
68
+ });
69
+
70
+ const finding = checkPausedModules(db);
71
+
72
+ expect(finding.status).toBe('fail');
73
+ expect(finding.summary).toContain('greenwave');
74
+ // The DURATION is what separates a maintenance window from a forgotten one.
75
+ expect(finding.detail.join('\n')).toContain('3d');
76
+ expect(finding.detail.join('\n')).toContain('edge router swap');
77
+ expect(finding.remediation).toContain('celilo module unpause');
78
+ });
79
+
80
+ test('there is no threshold — a pause taken seconds ago already fails', () => {
81
+ // Decided at review: a pause is a degraded state, full stop. A threshold is
82
+ // just something to tune until the detector stops firing.
83
+ const db = makeDb();
84
+ insert(db, 'greenwave', 'PAUSED', { pausedAt: new Date() });
85
+ expect(checkPausedModules(db).status).toBe('fail');
86
+ });
87
+ });
88
+
89
+ describe('listPausedModules / pausedAmong', () => {
90
+ test('reads only PAUSED rows, and reads back the reason', () => {
91
+ const db = makeDb();
92
+ insert(db, 'caddy', 'INSTALLED');
93
+ insert(db, 'greenwave', 'PAUSED', { reason: 'edge router swap' });
94
+ insert(db, 'authentik', 'ERROR');
95
+
96
+ const paused = listPausedModules(db);
97
+ expect(paused.map((m) => m.id)).toEqual(['greenwave']);
98
+ expect(paused[0].pauseReason).toBe('edge router swap');
99
+ });
100
+
101
+ test('an ERROR module is NOT paused — the two states are distinct', () => {
102
+ // Worth pinning: ERROR is *pausable*, which is easy to misread as "errored
103
+ // counts as paused" and would make the doctor check fire on every failure.
104
+ const db = makeDb();
105
+ insert(db, 'authentik', 'ERROR');
106
+ expect(listPausedModules(db)).toEqual([]);
107
+ });
108
+
109
+ test('pausedAmong filters a candidate set in one query', () => {
110
+ const db = makeDb();
111
+ insert(db, 'caddy', 'PAUSED');
112
+ insert(db, 'authentik', 'INSTALLED');
113
+ expect(pausedAmong(db, ['caddy', 'authentik', 'absent'])).toEqual(new Set(['caddy']));
114
+ });
115
+
116
+ test('pausedAmong on an empty list does not query at all', () => {
117
+ const db = makeDb();
118
+ expect(pausedAmong(db, [])).toEqual(new Set());
119
+ });
120
+ });
121
+
122
+ describe('the management-API warning is emitted, and NOT emitted (task 6.5 / 6.6)', () => {
123
+ // `fleetWarnings()` in api/serve.ts is the production caller; this pins the
124
+ // query + rendering it depends on. The inverse assertion is the one that
125
+ // stops the warning becoming noise nobody reads.
126
+ test('with nothing paused there is nothing to warn about', () => {
127
+ const db = makeDb();
128
+ insert(db, 'caddy', 'VERIFIED');
129
+ expect(listPausedModules(db)).toHaveLength(0);
130
+ });
131
+
132
+ test('with something paused the warning names the module AND its age', () => {
133
+ const db = makeDb();
134
+ insert(db, 'greenwave', 'PAUSED', {
135
+ pausedAt: new Date(Date.now() - 5 * 60 * 60 * 1000),
136
+ reason: 'edge router swap',
137
+ });
138
+
139
+ const rendered = listPausedModules(db).map((m) => describePausedModule(m));
140
+ expect(rendered).toEqual(['greenwave (5h, "edge router swap")']);
141
+ });
142
+
143
+ test('a pause with no reason still renders its age', () => {
144
+ const db = makeDb();
145
+ insert(db, 'greenwave', 'PAUSED', { pausedAt: new Date(Date.now() - 60 * 60 * 1000) });
146
+ expect(describePausedModule(listPausedModules(db)[0])).toBe('greenwave (1h)');
147
+ });
148
+ });
149
+
150
+ describe('a paused module is suppressed BY THE PAUSE, not anonymously (task 2.3)', () => {
151
+ const topology = { moduleSystems: [], zoneProviders: [] };
152
+
153
+ test('PROVE IT FIRES: an unpaused module with nothing firing is NOT suppressed', () => {
154
+ expect(
155
+ findSuppressor({
156
+ key: moduleAlertKey('caddy'),
157
+ firingKeys: new Set(),
158
+ suppressible: true,
159
+ modulesInDeployWindow: new Set(),
160
+ pausedModules: new Set(),
161
+ topology,
162
+ }),
163
+ ).toBeNull();
164
+ });
165
+
166
+ test('the same alert IS suppressed once the module is paused, attributed to the pause', () => {
167
+ expect(
168
+ findSuppressor({
169
+ key: moduleAlertKey('caddy'),
170
+ firingKeys: new Set(),
171
+ suppressible: true,
172
+ modulesInDeployWindow: new Set(),
173
+ pausedModules: new Set(['caddy']),
174
+ topology,
175
+ }),
176
+ ).toEqual({ kind: 'paused', moduleId: 'caddy' });
177
+ });
178
+
179
+ test('a pause does not suppress OTHER modules', () => {
180
+ expect(
181
+ findSuppressor({
182
+ key: moduleAlertKey('authentik'),
183
+ firingKeys: new Set(),
184
+ suppressible: true,
185
+ modulesInDeployWindow: new Set(),
186
+ pausedModules: new Set(['caddy']),
187
+ topology,
188
+ }),
189
+ ).toBeNull();
190
+ });
191
+
192
+ test('a self-monitor is never suppressed, even by a pause', () => {
193
+ // A cascading failure must not silence the component reporting the cascade.
194
+ expect(
195
+ findSuppressor({
196
+ key: moduleAlertKey('caddy'),
197
+ firingKeys: new Set(),
198
+ suppressible: false,
199
+ modulesInDeployWindow: new Set(),
200
+ pausedModules: new Set(['caddy']),
201
+ topology,
202
+ }),
203
+ ).toBeNull();
204
+ });
205
+ });
206
+
207
+ describe('--stop-infra acts only on celilo-provisioned infrastructure (design D2, revised)', () => {
208
+ // The RULING is about ownership, not capability, and the distinction lives in
209
+ // the message the operator reads — so the message is what gets pinned.
210
+ //
211
+ // celilo could SSH into a machine and stop something. It declines to, because
212
+ // a machine-pool system is operator-pre-provisioned, may predate celilo, and
213
+ // may run work celilo was never told about. Reporting "celilo cannot identify
214
+ // the service unit" would imply a capability gap where the truth is that this
215
+ // is not celilo's to stop.
216
+ test('the machine-hosted report reads as not-applicable, not as a missing feature', () => {
217
+ const report = describeMachineStopInfra('iot', 'homebridge');
218
+
219
+ expect(report).toContain('not applicable');
220
+ expect(report).toContain('celilo provisioned');
221
+ // Must NOT frame it as something celilo would do if only it could.
222
+ expect(report).not.toMatch(/cannot determine|does not know which|unit/i);
223
+ });
224
+ });