@celilo/cli 0.22.0 → 0.23.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CELILO_SUBSYSTEMS.md +34 -2
- package/drizzle/0024_module_pause.sql +20 -0
- package/drizzle/meta/_journal.json +8 -1
- package/package.json +4 -5
- package/src/__integration__/container-services-cli.integration.test.ts +8 -2
- package/src/api/remote-client.test.ts +6 -5
- package/src/api/serve.ts +41 -7
- package/src/api-clients/proxmox.ts +34 -0
- package/src/cli/commands/alerts-sweep.ts +2 -0
- package/src/cli/commands/events.ts +34 -3
- package/src/cli/commands/module-deploy.ts +2 -2
- package/src/cli/commands/module-health.ts +1 -0
- package/src/cli/commands/module-import.ts +3 -3
- package/src/cli/commands/module-list.ts +12 -1
- package/src/cli/commands/module-pause.ts +317 -0
- package/src/cli/commands/module-remove.ts +78 -40
- package/src/cli/commands/module-status.ts +3 -4
- package/src/cli/commands/module-update.test.ts +1 -1
- package/src/cli/commands/proxmox-template-selection.ts +1 -1
- package/src/cli/commands/status.ts +25 -3
- package/src/cli/completion.ts +4 -0
- package/src/cli/fuel-gauge.ts +4 -4
- package/src/cli/index.ts +45 -20
- package/src/cli/json-output.test.ts +162 -0
- package/src/cli/prompts.ts +53 -74
- package/src/cli/service-credential.ts +3 -3
- package/src/cli/stdout-is-undecorated.test.ts +94 -0
- package/src/cli/types.ts +7 -2
- package/src/db/schema.ts +73 -15
- package/src/hooks/run-named-hook.ts +28 -0
- package/src/services/alerting/suppression.test.ts +5 -0
- package/src/services/alerting/suppression.ts +18 -1
- package/src/services/alerting/sweep-runner.test.ts +1 -0
- package/src/services/alerting/sweep-runner.ts +11 -1
- package/src/services/bus-interview.ts +2 -2
- package/src/services/bus-secret-flow.test.ts +1 -1
- package/src/services/fleet-checks.ts +48 -0
- package/src/services/module-deploy.ts +1 -1
- package/src/services/module-pause-observability.test.ts +224 -0
- package/src/services/module-pause-quiescence.test.ts +163 -0
- package/src/services/module-pause.test.ts +573 -0
- package/src/services/module-pause.ts +544 -0
- package/src/services/remove-guard.test.ts +175 -0
- package/src/services/remove-guard.ts +109 -0
- package/src/services/terminal-responder.ts +16 -16
- package/src/services/update/dep-graph.test.ts +33 -4
- package/src/services/update/dep-graph.ts +39 -17
- package/src/services/zone-detector.ts +2 -39
- package/src/test-utils/cli.ts +15 -14
- package/src/test-utils/integration-guard.ts +26 -0
- package/src/test-utils/setup-test-db.ts +13 -23
package/src/db/schema.ts
CHANGED
|
@@ -11,7 +11,17 @@ import {
|
|
|
11
11
|
|
|
12
12
|
/**
|
|
13
13
|
* Module lifecycle states
|
|
14
|
-
* IMPORTED, VALIDATED, CONFIGURED, GENERATING, ERROR, DEPLOYING, INSTALLED, VERIFIED, UNINSTALLING
|
|
14
|
+
* IMPORTED, VALIDATED, CONFIGURED, GENERATING, ERROR, DEPLOYING, INSTALLED, VERIFIED, UNINSTALLING, PAUSED
|
|
15
|
+
*
|
|
16
|
+
* `PAUSED` is a real member of this union rather than a side flag, and that is
|
|
17
|
+
* the point (openspec/changes/module-pause-lifecycle/design.md D1): adding it
|
|
18
|
+
* makes the type-checker enumerate every site that must now consider
|
|
19
|
+
* paused-ness. A `pausedAt`-only flag would leave every `state === 'VERIFIED'`
|
|
20
|
+
* comparison silently compiling while quietly reading a paused module as live.
|
|
21
|
+
*
|
|
22
|
+
* There is deliberately no `prePauseState`: pause is legal only from a settled
|
|
23
|
+
* state and unpause redeploys, so the deploy path decides the resulting state
|
|
24
|
+
* and there is nothing to restore.
|
|
15
25
|
*/
|
|
16
26
|
export type ModuleState =
|
|
17
27
|
| 'IMPORTED'
|
|
@@ -22,23 +32,65 @@ export type ModuleState =
|
|
|
22
32
|
| 'INSTALLED'
|
|
23
33
|
| 'VERIFIED'
|
|
24
34
|
| 'ERROR'
|
|
25
|
-
| 'UNINSTALLING'
|
|
35
|
+
| 'UNINSTALLING'
|
|
36
|
+
| 'PAUSED';
|
|
37
|
+
|
|
38
|
+
/**
|
|
39
|
+
* States a module may be paused FROM (design D1). `ERROR` is deliberately
|
|
40
|
+
* included: quiescing a broken module to stop alert noise while working on it
|
|
41
|
+
* is legitimate, and refusing would push the operator toward silencing those
|
|
42
|
+
* alerts by some less visible route.
|
|
43
|
+
*/
|
|
44
|
+
export const PAUSABLE_STATES = [
|
|
45
|
+
'INSTALLED',
|
|
46
|
+
'VERIFIED',
|
|
47
|
+
'ERROR',
|
|
48
|
+
] as const satisfies readonly ModuleState[];
|
|
49
|
+
|
|
50
|
+
/**
|
|
51
|
+
* States that mean "a lifecycle transition is under way". Pausing one of these
|
|
52
|
+
* would strand the transition, so pause is refused with a distinct message from
|
|
53
|
+
* the never-deployed case.
|
|
54
|
+
*/
|
|
55
|
+
export const IN_FLIGHT_STATES = [
|
|
56
|
+
'GENERATING',
|
|
57
|
+
'DEPLOYING',
|
|
58
|
+
'UNINSTALLING',
|
|
59
|
+
] as const satisfies readonly ModuleState[];
|
|
26
60
|
|
|
27
61
|
/**
|
|
28
62
|
* Modules table - stores module metadata and manifest data
|
|
29
63
|
*/
|
|
30
|
-
export const modules = sqliteTable(
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
64
|
+
export const modules = sqliteTable(
|
|
65
|
+
'modules',
|
|
66
|
+
{
|
|
67
|
+
id: text('id').primaryKey(),
|
|
68
|
+
name: text('name').notNull(),
|
|
69
|
+
version: text('version').notNull(),
|
|
70
|
+
description: text('description'),
|
|
71
|
+
state: text('state').$type<ModuleState>().notNull().default('IMPORTED'),
|
|
72
|
+
manifestData: text('manifest_data', { mode: 'json' })
|
|
73
|
+
.$type<Record<string, unknown>>()
|
|
74
|
+
.notNull(),
|
|
75
|
+
sourcePath: text('source_path').notNull(),
|
|
76
|
+
importedAt: integer('imported_at', { mode: 'timestamp' }).notNull().default(sql`(unixepoch())`),
|
|
77
|
+
updatedAt: integer('updated_at', { mode: 'timestamp' }).notNull().default(sql`(unixepoch())`),
|
|
78
|
+
errorMessage: text('error_message'),
|
|
79
|
+
/**
|
|
80
|
+
* When the module was paused. Null unless `state = 'PAUSED'`. The state alone
|
|
81
|
+
* cannot answer "how long", and the DURATION is what makes a forgotten pause
|
|
82
|
+
* detectable (design D7) — every paused-module report carries the age.
|
|
83
|
+
*/
|
|
84
|
+
pausedAt: integer('paused_at', { mode: 'timestamp' }),
|
|
85
|
+
/** Operator-supplied explanation, so the row explains itself. */
|
|
86
|
+
pauseReason: text('pause_reason'),
|
|
87
|
+
},
|
|
88
|
+
(table) => ({
|
|
89
|
+
// Every management-API response asks "is anything paused?" (design D7), so
|
|
90
|
+
// that lookup must stay a single indexed hit rather than a table scan.
|
|
91
|
+
stateIdx: index('modules_state_idx').on(table.state),
|
|
92
|
+
}),
|
|
93
|
+
);
|
|
42
94
|
|
|
43
95
|
/**
|
|
44
96
|
* Module configuration - user-provided key-value pairs.
|
|
@@ -807,7 +859,13 @@ export const backups = sqliteTable('backups', {
|
|
|
807
859
|
* pid is no longer alive is treated as abandoned (the process crashed before
|
|
808
860
|
* the completion update landed) and ignored by in-flight checks.
|
|
809
861
|
*/
|
|
810
|
-
export type ModuleOperationKind =
|
|
862
|
+
export type ModuleOperationKind =
|
|
863
|
+
| 'deploy'
|
|
864
|
+
| 'uninstall'
|
|
865
|
+
| 'backup'
|
|
866
|
+
| 'restore'
|
|
867
|
+
| 'pause'
|
|
868
|
+
| 'unpause';
|
|
811
869
|
export type ModuleOperationStatus = 'in_progress' | 'completed' | 'failed';
|
|
812
870
|
|
|
813
871
|
export const moduleOperations = sqliteTable('module_operations', {
|
|
@@ -49,6 +49,13 @@ export interface RunNamedHookResult extends HookResult {
|
|
|
49
49
|
* gracefully when a module has no `on_uninstall` defined.
|
|
50
50
|
*/
|
|
51
51
|
notDefined?: boolean;
|
|
52
|
+
/**
|
|
53
|
+
* True when the hook was not run because the module is PAUSED. Reported as
|
|
54
|
+
* success rather than failure: the module is deliberately quiesced, and a
|
|
55
|
+
* failure here would be retried by the bus and then alerted on — paging the
|
|
56
|
+
* operator about the pause they took themselves.
|
|
57
|
+
*/
|
|
58
|
+
skippedPaused?: boolean;
|
|
52
59
|
}
|
|
53
60
|
|
|
54
61
|
/**
|
|
@@ -82,6 +89,27 @@ export async function runNamedHook(
|
|
|
82
89
|
};
|
|
83
90
|
}
|
|
84
91
|
|
|
92
|
+
// Quiescence for a paused module (openspec/changes/module-pause-lifecycle,
|
|
93
|
+
// tasks 2.1/2.2). This is the chokepoint every non-lifecycle invocation
|
|
94
|
+
// funnels through — bus dispatch, timer fan-out, aspect fan-out,
|
|
95
|
+
// public-web republish, the dns-provider backfill, and `module run-hook` —
|
|
96
|
+
// so guarding here covers the paths individually rather than each caller
|
|
97
|
+
// remembering to.
|
|
98
|
+
//
|
|
99
|
+
// The exemptions are decided from the hook NAME, not a caller-supplied flag
|
|
100
|
+
// (Rule 10.3): `on_install` is how unpause redeploys the module back to life,
|
|
101
|
+
// and `on_uninstall` is how a paused module is removed — which is the entire
|
|
102
|
+
// point of pausing it. Both must run while `state` is still PAUSED.
|
|
103
|
+
const LIFECYCLE_HOOKS: readonly HookName[] = ['on_install', 'on_uninstall'];
|
|
104
|
+
if (module.state === 'PAUSED' && !LIFECYCLE_HOOKS.includes(hookName)) {
|
|
105
|
+
return {
|
|
106
|
+
success: true,
|
|
107
|
+
outputs: {},
|
|
108
|
+
duration: Date.now() - startedAt,
|
|
109
|
+
skippedPaused: true,
|
|
110
|
+
};
|
|
111
|
+
}
|
|
112
|
+
|
|
85
113
|
const manifest = module.manifestData as ModuleManifest;
|
|
86
114
|
const hookDef = manifest.hooks?.[hookName as keyof typeof manifest.hooks];
|
|
87
115
|
if (!hookDef) {
|
|
@@ -35,6 +35,7 @@ const TOPOLOGY: SuppressionTopology = {
|
|
|
35
35
|
const noSuppression = {
|
|
36
36
|
suppressible: true,
|
|
37
37
|
modulesInDeployWindow: new Set<string>(),
|
|
38
|
+
pausedModules: new Set<string>(),
|
|
38
39
|
topology: TOPOLOGY,
|
|
39
40
|
};
|
|
40
41
|
|
|
@@ -185,6 +186,7 @@ describe('findSuppressor — guards', () => {
|
|
|
185
186
|
firingKeys,
|
|
186
187
|
suppressible: false,
|
|
187
188
|
modulesInDeployWindow: new Set(),
|
|
189
|
+
pausedModules: new Set(),
|
|
188
190
|
topology: TOPOLOGY,
|
|
189
191
|
}),
|
|
190
192
|
).toBeNull();
|
|
@@ -197,6 +199,7 @@ describe('findSuppressor — guards', () => {
|
|
|
197
199
|
firingKeys: new Set(),
|
|
198
200
|
suppressible: true,
|
|
199
201
|
modulesInDeployWindow: new Set(['forgejo']),
|
|
202
|
+
pausedModules: new Set(),
|
|
200
203
|
topology: TOPOLOGY,
|
|
201
204
|
}),
|
|
202
205
|
).toEqual({ kind: 'deploy_window', moduleId: 'forgejo' });
|
|
@@ -209,6 +212,7 @@ describe('findSuppressor — guards', () => {
|
|
|
209
212
|
firingKeys: new Set(),
|
|
210
213
|
suppressible: true,
|
|
211
214
|
modulesInDeployWindow: new Set(['forgejo']),
|
|
215
|
+
pausedModules: new Set(),
|
|
212
216
|
topology: TOPOLOGY,
|
|
213
217
|
}),
|
|
214
218
|
).toBeNull();
|
|
@@ -221,6 +225,7 @@ describe('findSuppressor — guards', () => {
|
|
|
221
225
|
firingKeys: new Set([machineAlertKey('iot')]),
|
|
222
226
|
suppressible: true,
|
|
223
227
|
modulesInDeployWindow: new Set(['homebridge']),
|
|
228
|
+
pausedModules: new Set(),
|
|
224
229
|
topology: TOPOLOGY,
|
|
225
230
|
}),
|
|
226
231
|
).toEqual({ kind: 'deploy_window', moduleId: 'homebridge' });
|
|
@@ -106,12 +106,20 @@ export interface SuppressorLookup {
|
|
|
106
106
|
suppressible: boolean;
|
|
107
107
|
/** Modules currently inside a deploy window. */
|
|
108
108
|
modulesInDeployWindow: ReadonlySet<string>;
|
|
109
|
+
/**
|
|
110
|
+
* Modules currently PAUSED. A pause is a deliberate quiescing, so its alerts
|
|
111
|
+
* are explained by the pause itself (openspec/changes/module-pause-lifecycle,
|
|
112
|
+
* task 2.3) — same mechanism as a deploy window, with the pause as the source
|
|
113
|
+
* instead of an ancestor alert.
|
|
114
|
+
*/
|
|
115
|
+
pausedModules: ReadonlySet<string>;
|
|
109
116
|
topology: SuppressionTopology;
|
|
110
117
|
}
|
|
111
118
|
|
|
112
119
|
export type Suppressor =
|
|
113
120
|
| { kind: 'alert'; key: string }
|
|
114
|
-
| { kind: 'deploy_window'; moduleId: string }
|
|
121
|
+
| { kind: 'deploy_window'; moduleId: string }
|
|
122
|
+
| { kind: 'paused'; moduleId: string };
|
|
115
123
|
|
|
116
124
|
/**
|
|
117
125
|
* Find what is suppressing `key`, or null if it should be reported.
|
|
@@ -128,6 +136,15 @@ export function findSuppressor(lookup: SuppressorLookup): Suppressor | null {
|
|
|
128
136
|
|
|
129
137
|
const parsed = parseAlertKey(lookup.key);
|
|
130
138
|
|
|
139
|
+
// A pause is checked before a deploy window because it is the longer-lived
|
|
140
|
+
// and more consequential explanation: a paused module may also be inside a
|
|
141
|
+
// deploy window (unpause redeploys), and "paused" is the fact the operator
|
|
142
|
+
// needs to see. Attributed to the pause rather than suppressed anonymously —
|
|
143
|
+
// silently dropping the alert is what turns a pause into an invisible outage.
|
|
144
|
+
if (parsed?.source === 'module' && lookup.pausedModules.has(parsed.moduleId)) {
|
|
145
|
+
return { kind: 'paused', moduleId: parsed.moduleId };
|
|
146
|
+
}
|
|
147
|
+
|
|
131
148
|
// A deploy is the same mechanism with a window as the source instead of an
|
|
132
149
|
// ancestor alert — which is why deploy auto-silencing is not a second feature.
|
|
133
150
|
if (parsed?.source === 'module' && lookup.modulesInDeployWindow.has(parsed.moduleId)) {
|
|
@@ -50,6 +50,7 @@ describe('runSweep', () => {
|
|
|
50
50
|
monitorDeps: monitorDeps(result, now),
|
|
51
51
|
loadTopology: () => TOPOLOGY,
|
|
52
52
|
loadDeployWindowModules: () => new Set(),
|
|
53
|
+
loadPausedModules: () => new Set(),
|
|
53
54
|
isSuppressible: () => true,
|
|
54
55
|
// No routes configured: the sweep must still run everything else.
|
|
55
56
|
notifyDepsFor: () => null,
|
|
@@ -43,6 +43,8 @@ export interface SweepDeps {
|
|
|
43
43
|
loadTopology(): SuppressionTopology;
|
|
44
44
|
/** Modules currently inside a deploy window. */
|
|
45
45
|
loadDeployWindowModules(): Set<string>;
|
|
46
|
+
/** Ids of modules currently PAUSED — a pause explains its own module's alerts. */
|
|
47
|
+
loadPausedModules(): Set<string>;
|
|
46
48
|
/** Whether the monitor owning an alert may be suppressed at all. */
|
|
47
49
|
isSuppressible(alert: Alert): boolean;
|
|
48
50
|
/** Compose the per-alert notification context. Null when nothing can page. */
|
|
@@ -156,6 +158,7 @@ export async function runSweep(
|
|
|
156
158
|
);
|
|
157
159
|
const topology = deps.loadTopology();
|
|
158
160
|
const deployWindows = deps.loadDeployWindowModules();
|
|
161
|
+
const paused = deps.loadPausedModules();
|
|
159
162
|
|
|
160
163
|
for (const alert of live) {
|
|
161
164
|
const suppressor = findSuppressor({
|
|
@@ -163,13 +166,20 @@ export async function runSweep(
|
|
|
163
166
|
firingKeys,
|
|
164
167
|
suppressible: deps.isSuppressible(alert),
|
|
165
168
|
modulesInDeployWindow: deployWindows,
|
|
169
|
+
pausedModules: paused,
|
|
166
170
|
topology,
|
|
167
171
|
});
|
|
168
172
|
const wasSuppressed = alert.state === 'suppressed';
|
|
169
173
|
if (suppressor && !wasSuppressed) {
|
|
170
174
|
markSuppressed(db, alert.id, {
|
|
171
175
|
alertId: suppressor.kind === 'alert' ? suppressor.key : undefined,
|
|
172
|
-
|
|
176
|
+
// A pause is recorded on the same column as a deploy window: both are
|
|
177
|
+
// "a module-scoped condition explains this", and the operator reads the
|
|
178
|
+
// module id either way.
|
|
179
|
+
windowId:
|
|
180
|
+
suppressor.kind === 'deploy_window' || suppressor.kind === 'paused'
|
|
181
|
+
? suppressor.moduleId
|
|
182
|
+
: undefined,
|
|
173
183
|
});
|
|
174
184
|
report.suppressed++;
|
|
175
185
|
} else if (!suppressor && wasSuppressed) {
|
|
@@ -314,7 +314,7 @@ export async function busInterviewGuarded<TReply>(
|
|
|
314
314
|
* Ask a single generic interview question over the bus and return the
|
|
315
315
|
* responder's answer (ISS-0127). The generic counterpart to the deploy's
|
|
316
316
|
* config/secret/ensure interview: any operator command can call this to make
|
|
317
|
-
* its prompts headlessly drivable instead of
|
|
317
|
+
* its prompts headlessly drivable instead of prompting on stdin directly.
|
|
318
318
|
*
|
|
319
319
|
* The return type is `unknown` because the runtime shape depends on
|
|
320
320
|
* `payload.kind`; prefer the typed wrappers (`askText`, `askSelect`,
|
|
@@ -345,7 +345,7 @@ export async function askInterview(
|
|
|
345
345
|
*
|
|
346
346
|
* This is the shared lifecycle every migrated command wraps its interview in,
|
|
347
347
|
* so the start/close boilerplate lives in exactly one place. The dynamic
|
|
348
|
-
* import keeps
|
|
348
|
+
* import keeps the terminal renderer out of the non-TTY path's module graph.
|
|
349
349
|
*/
|
|
350
350
|
export async function withInterviewSession<T>(fn: () => Promise<T>): Promise<T> {
|
|
351
351
|
const responder = process.stdin.isTTY
|
|
@@ -3,7 +3,7 @@
|
|
|
3
3
|
*
|
|
4
4
|
* Drives `interviewForMissingSecrets` against a real sqlite bus + real
|
|
5
5
|
* encrypted store + a programmatic test responder. No fixture modules,
|
|
6
|
-
* no machines, no
|
|
6
|
+
* no machines, no terminal — the responder is just a `bus.watch` that
|
|
7
7
|
* mimics what `terminal-responder.ts` does.
|
|
8
8
|
*
|
|
9
9
|
* Covers what stage 3 introduced:
|
|
@@ -38,6 +38,7 @@ import {
|
|
|
38
38
|
readInstalledUnit,
|
|
39
39
|
unitMainPid,
|
|
40
40
|
} from './events-daemon';
|
|
41
|
+
import { describePausedModule, listPausedModules } from './module-pause';
|
|
41
42
|
import { resolveSubscription } from './module-subscriptions';
|
|
42
43
|
|
|
43
44
|
/**
|
|
@@ -854,6 +855,52 @@ export function checkControlPlaneNetwork(db: DbClient): FleetFinding {
|
|
|
854
855
|
};
|
|
855
856
|
}
|
|
856
857
|
|
|
858
|
+
/**
|
|
859
|
+
* Any paused module is a doctor failure — no threshold, whatever its age
|
|
860
|
+
* (openspec/changes/module-pause-lifecycle, design D7, closed at review).
|
|
861
|
+
*
|
|
862
|
+
* A pause deliberately switches OFF the alerting that would otherwise report
|
|
863
|
+
* the module as down, so the paused-ness itself has to be the signal. A
|
|
864
|
+
* duration threshold was considered and dropped: there is no number of hours
|
|
865
|
+
* after which a deliberate outage becomes acceptable, and a configurable one is
|
|
866
|
+
* just an invitation to tune the detector until it stops firing.
|
|
867
|
+
*
|
|
868
|
+
* The fleet had just run 20 hours of failing forgejo backups whose only symptom
|
|
869
|
+
* was a column of `0 B` rows that read as healthy hourly cadence. Same failure
|
|
870
|
+
* shape; this is the check that would have named it.
|
|
871
|
+
*/
|
|
872
|
+
export function checkPausedModules(db: DbClient): FleetFinding {
|
|
873
|
+
const paused = listPausedModules(db);
|
|
874
|
+
|
|
875
|
+
if (paused.length === 0) {
|
|
876
|
+
return {
|
|
877
|
+
id: 'paused-modules',
|
|
878
|
+
title: 'No module is paused (a pause suppresses its own alerting)',
|
|
879
|
+
status: 'ok',
|
|
880
|
+
summary: 'nothing paused',
|
|
881
|
+
detail: [],
|
|
882
|
+
remediation: null,
|
|
883
|
+
autoFixable: false,
|
|
884
|
+
};
|
|
885
|
+
}
|
|
886
|
+
|
|
887
|
+
const names = paused.map((m) => describePausedModule(m));
|
|
888
|
+
return {
|
|
889
|
+
id: 'paused-modules',
|
|
890
|
+
title: 'No module is paused (a pause suppresses its own alerting)',
|
|
891
|
+
status: 'fail',
|
|
892
|
+
summary: `${paused.length} module(s) paused: ${paused.map((m) => m.id).join(', ')}`,
|
|
893
|
+
detail: [
|
|
894
|
+
...names.map((n) => `paused: ${n}`),
|
|
895
|
+
'a paused module receives no dispatched work, runs no health checks, and has its alerts suppressed',
|
|
896
|
+
'it is still deployed and may still be serving traffic — pause does not stop the data plane',
|
|
897
|
+
],
|
|
898
|
+
remediation:
|
|
899
|
+
'bring each back with "celilo module unpause <id>" (which redeploys it, rebinding its capabilities), or remove it if the pause was permanent',
|
|
900
|
+
autoFixable: false,
|
|
901
|
+
};
|
|
902
|
+
}
|
|
903
|
+
|
|
857
904
|
export async function runFleetChecks(
|
|
858
905
|
bus: Bus,
|
|
859
906
|
db: DbClient,
|
|
@@ -865,6 +912,7 @@ export async function runFleetChecks(
|
|
|
865
912
|
checkSubscribers(bus, db),
|
|
866
913
|
checkCapabilityProviders(db),
|
|
867
914
|
checkControlPlaneNetwork(db),
|
|
915
|
+
checkPausedModules(db),
|
|
868
916
|
await checkServiceDns(db),
|
|
869
917
|
];
|
|
870
918
|
}
|
|
@@ -350,7 +350,7 @@ async function deployModuleImpl(
|
|
|
350
350
|
|
|
351
351
|
// Terminal-responder: when running on a TTY, this subscribes to
|
|
352
352
|
// `config.required.*` / `secret.required.*` / `ensure.required.*`
|
|
353
|
-
// events and prompts
|
|
353
|
+
// events and prompts on the terminal. Other responder shapes (Claude
|
|
354
354
|
// subagent, `celilo events respond` from another shell, autoresponder
|
|
355
355
|
// daemon) compete on the bus; first reply wins. See
|
|
356
356
|
// infra/openspec/changes/interactive-deploys-via-event-bus/proposal.md.
|
|
@@ -0,0 +1,224 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The parts of pause that exist so a pause cannot become a silent outage
|
|
3
|
+
* (design D7), plus the alerting suppressor.
|
|
4
|
+
*
|
|
5
|
+
* Every check here is asserted in BOTH directions per Rule 7.6 — a gate nobody
|
|
6
|
+
* has seen fail is not a gate, and "no paused module was reported" is exactly
|
|
7
|
+
* the output a broken detector produces.
|
|
8
|
+
*/
|
|
9
|
+
|
|
10
|
+
import { describe, expect, test } from 'bun:test';
|
|
11
|
+
import { mkdtempSync } from 'node:fs';
|
|
12
|
+
import { tmpdir } from 'node:os';
|
|
13
|
+
import { join } from 'node:path';
|
|
14
|
+
import { type DbClient, createDbClient } from '../db/client';
|
|
15
|
+
import { type ModuleState, modules } from '../db/schema';
|
|
16
|
+
import { moduleAlertKey } from './alerting/keys';
|
|
17
|
+
import { findSuppressor } from './alerting/suppression';
|
|
18
|
+
import { checkPausedModules } from './fleet-checks';
|
|
19
|
+
import {
|
|
20
|
+
describeMachineStopInfra,
|
|
21
|
+
describePausedModule,
|
|
22
|
+
listPausedModules,
|
|
23
|
+
pausedAmong,
|
|
24
|
+
} from './module-pause';
|
|
25
|
+
|
|
26
|
+
function makeDb(): DbClient {
|
|
27
|
+
const dir = mkdtempSync(join(tmpdir(), 'celilo-pause-obs-'));
|
|
28
|
+
process.env.CELILO_DB_PATH = join(dir, 'celilo.db');
|
|
29
|
+
return createDbClient({ path: process.env.CELILO_DB_PATH });
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
function insert(
|
|
33
|
+
db: DbClient,
|
|
34
|
+
id: string,
|
|
35
|
+
state: ModuleState,
|
|
36
|
+
opts: { pausedAt?: Date; reason?: string } = {},
|
|
37
|
+
): void {
|
|
38
|
+
db.insert(modules)
|
|
39
|
+
.values({
|
|
40
|
+
id,
|
|
41
|
+
name: id,
|
|
42
|
+
version: '1.0.0',
|
|
43
|
+
state,
|
|
44
|
+
manifestData: { id, name: id, version: '1.0.0' },
|
|
45
|
+
sourcePath: `/tmp/${id}`,
|
|
46
|
+
pausedAt: state === 'PAUSED' ? (opts.pausedAt ?? new Date()) : null,
|
|
47
|
+
pauseReason: state === 'PAUSED' ? (opts.reason ?? null) : null,
|
|
48
|
+
})
|
|
49
|
+
.run();
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
describe('system doctor reports ANY paused module (task 6.2 / 6.4)', () => {
|
|
53
|
+
test('with nothing paused the check passes', () => {
|
|
54
|
+
const db = makeDb();
|
|
55
|
+
insert(db, 'caddy', 'VERIFIED');
|
|
56
|
+
|
|
57
|
+
const finding = checkPausedModules(db);
|
|
58
|
+
expect(finding.status).toBe('ok');
|
|
59
|
+
expect(finding.summary).toBe('nothing paused');
|
|
60
|
+
});
|
|
61
|
+
|
|
62
|
+
test('PROVE IT FAILS: one paused module is a doctor FAILURE naming it and its age', () => {
|
|
63
|
+
const db = makeDb();
|
|
64
|
+
insert(db, 'caddy', 'VERIFIED');
|
|
65
|
+
insert(db, 'greenwave', 'PAUSED', {
|
|
66
|
+
pausedAt: new Date(Date.now() - 3 * 24 * 60 * 60 * 1000),
|
|
67
|
+
reason: 'edge router swap',
|
|
68
|
+
});
|
|
69
|
+
|
|
70
|
+
const finding = checkPausedModules(db);
|
|
71
|
+
|
|
72
|
+
expect(finding.status).toBe('fail');
|
|
73
|
+
expect(finding.summary).toContain('greenwave');
|
|
74
|
+
// The DURATION is what separates a maintenance window from a forgotten one.
|
|
75
|
+
expect(finding.detail.join('\n')).toContain('3d');
|
|
76
|
+
expect(finding.detail.join('\n')).toContain('edge router swap');
|
|
77
|
+
expect(finding.remediation).toContain('celilo module unpause');
|
|
78
|
+
});
|
|
79
|
+
|
|
80
|
+
test('there is no threshold — a pause taken seconds ago already fails', () => {
|
|
81
|
+
// Decided at review: a pause is a degraded state, full stop. A threshold is
|
|
82
|
+
// just something to tune until the detector stops firing.
|
|
83
|
+
const db = makeDb();
|
|
84
|
+
insert(db, 'greenwave', 'PAUSED', { pausedAt: new Date() });
|
|
85
|
+
expect(checkPausedModules(db).status).toBe('fail');
|
|
86
|
+
});
|
|
87
|
+
});
|
|
88
|
+
|
|
89
|
+
describe('listPausedModules / pausedAmong', () => {
|
|
90
|
+
test('reads only PAUSED rows, and reads back the reason', () => {
|
|
91
|
+
const db = makeDb();
|
|
92
|
+
insert(db, 'caddy', 'INSTALLED');
|
|
93
|
+
insert(db, 'greenwave', 'PAUSED', { reason: 'edge router swap' });
|
|
94
|
+
insert(db, 'authentik', 'ERROR');
|
|
95
|
+
|
|
96
|
+
const paused = listPausedModules(db);
|
|
97
|
+
expect(paused.map((m) => m.id)).toEqual(['greenwave']);
|
|
98
|
+
expect(paused[0].pauseReason).toBe('edge router swap');
|
|
99
|
+
});
|
|
100
|
+
|
|
101
|
+
test('an ERROR module is NOT paused — the two states are distinct', () => {
|
|
102
|
+
// Worth pinning: ERROR is *pausable*, which is easy to misread as "errored
|
|
103
|
+
// counts as paused" and would make the doctor check fire on every failure.
|
|
104
|
+
const db = makeDb();
|
|
105
|
+
insert(db, 'authentik', 'ERROR');
|
|
106
|
+
expect(listPausedModules(db)).toEqual([]);
|
|
107
|
+
});
|
|
108
|
+
|
|
109
|
+
test('pausedAmong filters a candidate set in one query', () => {
|
|
110
|
+
const db = makeDb();
|
|
111
|
+
insert(db, 'caddy', 'PAUSED');
|
|
112
|
+
insert(db, 'authentik', 'INSTALLED');
|
|
113
|
+
expect(pausedAmong(db, ['caddy', 'authentik', 'absent'])).toEqual(new Set(['caddy']));
|
|
114
|
+
});
|
|
115
|
+
|
|
116
|
+
test('pausedAmong on an empty list does not query at all', () => {
|
|
117
|
+
const db = makeDb();
|
|
118
|
+
expect(pausedAmong(db, [])).toEqual(new Set());
|
|
119
|
+
});
|
|
120
|
+
});
|
|
121
|
+
|
|
122
|
+
describe('the management-API warning is emitted, and NOT emitted (task 6.5 / 6.6)', () => {
|
|
123
|
+
// `fleetWarnings()` in api/serve.ts is the production caller; this pins the
|
|
124
|
+
// query + rendering it depends on. The inverse assertion is the one that
|
|
125
|
+
// stops the warning becoming noise nobody reads.
|
|
126
|
+
test('with nothing paused there is nothing to warn about', () => {
|
|
127
|
+
const db = makeDb();
|
|
128
|
+
insert(db, 'caddy', 'VERIFIED');
|
|
129
|
+
expect(listPausedModules(db)).toHaveLength(0);
|
|
130
|
+
});
|
|
131
|
+
|
|
132
|
+
test('with something paused the warning names the module AND its age', () => {
|
|
133
|
+
const db = makeDb();
|
|
134
|
+
insert(db, 'greenwave', 'PAUSED', {
|
|
135
|
+
pausedAt: new Date(Date.now() - 5 * 60 * 60 * 1000),
|
|
136
|
+
reason: 'edge router swap',
|
|
137
|
+
});
|
|
138
|
+
|
|
139
|
+
const rendered = listPausedModules(db).map((m) => describePausedModule(m));
|
|
140
|
+
expect(rendered).toEqual(['greenwave (5h, "edge router swap")']);
|
|
141
|
+
});
|
|
142
|
+
|
|
143
|
+
test('a pause with no reason still renders its age', () => {
|
|
144
|
+
const db = makeDb();
|
|
145
|
+
insert(db, 'greenwave', 'PAUSED', { pausedAt: new Date(Date.now() - 60 * 60 * 1000) });
|
|
146
|
+
expect(describePausedModule(listPausedModules(db)[0])).toBe('greenwave (1h)');
|
|
147
|
+
});
|
|
148
|
+
});
|
|
149
|
+
|
|
150
|
+
describe('a paused module is suppressed BY THE PAUSE, not anonymously (task 2.3)', () => {
|
|
151
|
+
const topology = { moduleSystems: [], zoneProviders: [] };
|
|
152
|
+
|
|
153
|
+
test('PROVE IT FIRES: an unpaused module with nothing firing is NOT suppressed', () => {
|
|
154
|
+
expect(
|
|
155
|
+
findSuppressor({
|
|
156
|
+
key: moduleAlertKey('caddy'),
|
|
157
|
+
firingKeys: new Set(),
|
|
158
|
+
suppressible: true,
|
|
159
|
+
modulesInDeployWindow: new Set(),
|
|
160
|
+
pausedModules: new Set(),
|
|
161
|
+
topology,
|
|
162
|
+
}),
|
|
163
|
+
).toBeNull();
|
|
164
|
+
});
|
|
165
|
+
|
|
166
|
+
test('the same alert IS suppressed once the module is paused, attributed to the pause', () => {
|
|
167
|
+
expect(
|
|
168
|
+
findSuppressor({
|
|
169
|
+
key: moduleAlertKey('caddy'),
|
|
170
|
+
firingKeys: new Set(),
|
|
171
|
+
suppressible: true,
|
|
172
|
+
modulesInDeployWindow: new Set(),
|
|
173
|
+
pausedModules: new Set(['caddy']),
|
|
174
|
+
topology,
|
|
175
|
+
}),
|
|
176
|
+
).toEqual({ kind: 'paused', moduleId: 'caddy' });
|
|
177
|
+
});
|
|
178
|
+
|
|
179
|
+
test('a pause does not suppress OTHER modules', () => {
|
|
180
|
+
expect(
|
|
181
|
+
findSuppressor({
|
|
182
|
+
key: moduleAlertKey('authentik'),
|
|
183
|
+
firingKeys: new Set(),
|
|
184
|
+
suppressible: true,
|
|
185
|
+
modulesInDeployWindow: new Set(),
|
|
186
|
+
pausedModules: new Set(['caddy']),
|
|
187
|
+
topology,
|
|
188
|
+
}),
|
|
189
|
+
).toBeNull();
|
|
190
|
+
});
|
|
191
|
+
|
|
192
|
+
test('a self-monitor is never suppressed, even by a pause', () => {
|
|
193
|
+
// A cascading failure must not silence the component reporting the cascade.
|
|
194
|
+
expect(
|
|
195
|
+
findSuppressor({
|
|
196
|
+
key: moduleAlertKey('caddy'),
|
|
197
|
+
firingKeys: new Set(),
|
|
198
|
+
suppressible: false,
|
|
199
|
+
modulesInDeployWindow: new Set(),
|
|
200
|
+
pausedModules: new Set(['caddy']),
|
|
201
|
+
topology,
|
|
202
|
+
}),
|
|
203
|
+
).toBeNull();
|
|
204
|
+
});
|
|
205
|
+
});
|
|
206
|
+
|
|
207
|
+
describe('--stop-infra acts only on celilo-provisioned infrastructure (design D2, revised)', () => {
|
|
208
|
+
// The RULING is about ownership, not capability, and the distinction lives in
|
|
209
|
+
// the message the operator reads — so the message is what gets pinned.
|
|
210
|
+
//
|
|
211
|
+
// celilo could SSH into a machine and stop something. It declines to, because
|
|
212
|
+
// a machine-pool system is operator-pre-provisioned, may predate celilo, and
|
|
213
|
+
// may run work celilo was never told about. Reporting "celilo cannot identify
|
|
214
|
+
// the service unit" would imply a capability gap where the truth is that this
|
|
215
|
+
// is not celilo's to stop.
|
|
216
|
+
test('the machine-hosted report reads as not-applicable, not as a missing feature', () => {
|
|
217
|
+
const report = describeMachineStopInfra('iot', 'homebridge');
|
|
218
|
+
|
|
219
|
+
expect(report).toContain('not applicable');
|
|
220
|
+
expect(report).toContain('celilo provisioned');
|
|
221
|
+
// Must NOT frame it as something celilo would do if only it could.
|
|
222
|
+
expect(report).not.toMatch(/cannot determine|does not know which|unit/i);
|
|
223
|
+
});
|
|
224
|
+
});
|