@celilo/cli 0.21.0 → 0.23.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CELILO_CORE_MODULES.md +5 -4
- package/CELILO_SUBSYSTEMS.md +34 -2
- package/drizzle/0024_module_pause.sql +20 -0
- package/drizzle/meta/_journal.json +8 -1
- package/package.json +5 -6
- package/src/__integration__/container-services-cli.integration.test.ts +8 -2
- package/src/api/remote-client.test.ts +6 -5
- package/src/api/serve.ts +41 -7
- package/src/api-clients/proxmox.ts +34 -0
- package/src/cli/commands/alerts-sweep.ts +2 -0
- package/src/cli/commands/events.test.ts +66 -0
- package/src/cli/commands/events.ts +106 -3
- package/src/cli/commands/module-deploy.ts +2 -2
- package/src/cli/commands/module-health.ts +1 -0
- package/src/cli/commands/module-import.ts +3 -3
- package/src/cli/commands/module-list.ts +12 -1
- package/src/cli/commands/module-pause.ts +317 -0
- package/src/cli/commands/module-remove.ts +78 -40
- package/src/cli/commands/module-status.ts +3 -4
- package/src/cli/commands/module-update.test.ts +1 -1
- package/src/cli/commands/proxmox-template-selection.ts +1 -1
- package/src/cli/commands/status.ts +25 -3
- package/src/cli/completion.ts +5 -0
- package/src/cli/fuel-gauge.ts +4 -4
- package/src/cli/index.ts +49 -20
- package/src/cli/json-output.test.ts +162 -0
- package/src/cli/prompts.ts +53 -74
- package/src/cli/service-credential.ts +3 -3
- package/src/cli/stdout-is-undecorated.test.ts +94 -0
- package/src/cli/types.ts +7 -2
- package/src/db/schema.ts +73 -15
- package/src/hooks/run-named-hook.ts +28 -0
- package/src/services/alerting/suppression.test.ts +5 -0
- package/src/services/alerting/suppression.ts +18 -1
- package/src/services/alerting/sweep-runner.test.ts +1 -0
- package/src/services/alerting/sweep-runner.ts +11 -1
- package/src/services/bus-interview.ts +2 -2
- package/src/services/bus-secret-flow.test.ts +1 -1
- package/src/services/dns-registrations.ts +12 -0
- package/src/services/fleet-checks.test.ts +46 -0
- package/src/services/fleet-checks.ts +63 -6
- package/src/services/module-deploy.ts +1 -1
- package/src/services/module-pause-observability.test.ts +224 -0
- package/src/services/module-pause-quiescence.test.ts +163 -0
- package/src/services/module-pause.test.ts +573 -0
- package/src/services/module-pause.ts +544 -0
- package/src/services/remove-guard.test.ts +175 -0
- package/src/services/remove-guard.ts +109 -0
- package/src/services/terminal-responder.ts +16 -16
- package/src/services/update/dep-graph.test.ts +33 -4
- package/src/services/update/dep-graph.ts +39 -17
- package/src/services/zone-detector.ts +2 -39
- package/src/test-utils/cli.ts +15 -14
- package/src/test-utils/integration-guard.ts +26 -0
- package/src/test-utils/setup-test-db.ts +13 -23
package/src/cli/prompts.ts
CHANGED
|
@@ -1,10 +1,32 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* Prompt Utilities
|
|
3
|
-
*
|
|
3
|
+
*
|
|
4
|
+
* A thin celilo-facing wrapper over the prompt/message primitives in
|
|
5
|
+
* `@celilo/cli-display`. Roughly 160 `log.*` call sites and every legacy
|
|
6
|
+
* direct prompt route through this module, which is what made replacing
|
|
7
|
+
* `@clack/prompts` a one-file change rather than a 160-file one (celilo#699).
|
|
8
|
+
*
|
|
9
|
+
* NOTE: new interactive decisions must NOT be added here. They belong on the
|
|
10
|
+
* event-bus interview (`services/bus-interview`), which a headless responder
|
|
11
|
+
* can answer; `test-integration/cli/headless-drivable.test.ts` enforces that
|
|
12
|
+
* no command reaches for these directly. What remains is the terminal renderer
|
|
13
|
+
* for a bus question, plus the legacy prompts that predate the interview.
|
|
4
14
|
*/
|
|
5
15
|
|
|
6
|
-
import {
|
|
7
|
-
|
|
16
|
+
import {
|
|
17
|
+
CANCEL,
|
|
18
|
+
cancel,
|
|
19
|
+
getActiveDisplay,
|
|
20
|
+
intro,
|
|
21
|
+
isCancel,
|
|
22
|
+
log,
|
|
23
|
+
note,
|
|
24
|
+
outro,
|
|
25
|
+
password,
|
|
26
|
+
setActiveDisplay,
|
|
27
|
+
text,
|
|
28
|
+
confirm as uiConfirm,
|
|
29
|
+
} from '@celilo/cli-display';
|
|
8
30
|
|
|
9
31
|
// Re-export so existing imports from `../cli/prompts` keep working.
|
|
10
32
|
// The singleton itself lives in @celilo/cli-display so module code
|
|
@@ -12,25 +34,26 @@ import * as p from '@clack/prompts';
|
|
|
12
34
|
// it via @celilo/capabilities's re-export.
|
|
13
35
|
export { getActiveDisplay, setActiveDisplay };
|
|
14
36
|
|
|
37
|
+
export { log };
|
|
38
|
+
|
|
15
39
|
/**
|
|
16
40
|
* Interactive prompt wrapper with celilo branding
|
|
17
41
|
*/
|
|
18
42
|
export async function celiloIntro(title: string): Promise<void> {
|
|
19
|
-
|
|
43
|
+
intro(title);
|
|
20
44
|
}
|
|
21
45
|
|
|
22
46
|
export async function celiloOutro(message: string): Promise<void> {
|
|
23
|
-
|
|
47
|
+
outro(message);
|
|
24
48
|
}
|
|
25
49
|
|
|
26
50
|
/**
|
|
27
|
-
* Prompt for text input with validation and help text
|
|
51
|
+
* Prompt for text input with validation and help text.
|
|
28
52
|
*
|
|
29
|
-
* Displays
|
|
30
|
-
* -
|
|
31
|
-
*
|
|
32
|
-
*
|
|
33
|
-
* - Validation errors inline
|
|
53
|
+
* Displays the question, an input field with the placeholder, and any
|
|
54
|
+
* validation error inline. A cancel (Ctrl-C / Escape) exits the process, which
|
|
55
|
+
* is the behaviour every existing caller is written against: they treat the
|
|
56
|
+
* return as an unconditional `string`.
|
|
34
57
|
*/
|
|
35
58
|
export async function promptText(options: {
|
|
36
59
|
message: string;
|
|
@@ -38,19 +61,14 @@ export async function promptText(options: {
|
|
|
38
61
|
placeholder?: string;
|
|
39
62
|
validate?: (value: string | undefined) => string | Error | undefined;
|
|
40
63
|
}): Promise<string> {
|
|
41
|
-
const result = await
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
validate: options.validate,
|
|
46
|
-
});
|
|
47
|
-
|
|
48
|
-
if (p.isCancel(result)) {
|
|
49
|
-
p.cancel('Operation cancelled');
|
|
64
|
+
const result = await text(options);
|
|
65
|
+
|
|
66
|
+
if (isCancel(result)) {
|
|
67
|
+
cancel('Operation cancelled');
|
|
50
68
|
process.exit(0);
|
|
51
69
|
}
|
|
52
70
|
|
|
53
|
-
return result
|
|
71
|
+
return result;
|
|
54
72
|
}
|
|
55
73
|
|
|
56
74
|
/**
|
|
@@ -62,30 +80,25 @@ export async function promptPassword(options: {
|
|
|
62
80
|
validate?: (value: string | undefined) => string | Error | undefined;
|
|
63
81
|
}): Promise<string> {
|
|
64
82
|
while (true) {
|
|
65
|
-
const result = await
|
|
66
|
-
message: options.message,
|
|
67
|
-
validate: options.validate,
|
|
68
|
-
});
|
|
83
|
+
const result = await password(options);
|
|
69
84
|
|
|
70
|
-
if (
|
|
71
|
-
|
|
85
|
+
if (isCancel(result)) {
|
|
86
|
+
cancel('Operation cancelled');
|
|
72
87
|
process.exit(0);
|
|
73
88
|
}
|
|
74
89
|
|
|
75
|
-
const
|
|
76
|
-
message: `Confirm ${options.message}`,
|
|
77
|
-
});
|
|
90
|
+
const confirmation = await password({ message: `Confirm ${options.message}` });
|
|
78
91
|
|
|
79
|
-
if (
|
|
80
|
-
|
|
92
|
+
if (isCancel(confirmation)) {
|
|
93
|
+
cancel('Operation cancelled');
|
|
81
94
|
process.exit(0);
|
|
82
95
|
}
|
|
83
96
|
|
|
84
|
-
if (result ===
|
|
85
|
-
return result
|
|
97
|
+
if (result === confirmation) {
|
|
98
|
+
return result;
|
|
86
99
|
}
|
|
87
100
|
|
|
88
|
-
|
|
101
|
+
log.warn('Passwords do not match. Please try again.');
|
|
89
102
|
}
|
|
90
103
|
}
|
|
91
104
|
|
|
@@ -96,53 +109,19 @@ export async function promptConfirm(options: {
|
|
|
96
109
|
message: string;
|
|
97
110
|
initialValue?: boolean;
|
|
98
111
|
}): Promise<boolean> {
|
|
99
|
-
const result = await
|
|
100
|
-
message: options.message,
|
|
101
|
-
initialValue: options.initialValue,
|
|
102
|
-
});
|
|
112
|
+
const result = await uiConfirm(options);
|
|
103
113
|
|
|
104
|
-
if (
|
|
105
|
-
|
|
114
|
+
if (result === CANCEL) {
|
|
115
|
+
cancel('Operation cancelled');
|
|
106
116
|
process.exit(0);
|
|
107
117
|
}
|
|
108
118
|
|
|
109
|
-
return result
|
|
119
|
+
return result;
|
|
110
120
|
}
|
|
111
121
|
|
|
112
122
|
/**
|
|
113
123
|
* Show note/information box
|
|
114
124
|
*/
|
|
115
125
|
export function showNote(message: string, title?: string): void {
|
|
116
|
-
|
|
126
|
+
note(message, title);
|
|
117
127
|
}
|
|
118
|
-
|
|
119
|
-
/**
|
|
120
|
-
* Show log messages
|
|
121
|
-
*/
|
|
122
|
-
export const log = {
|
|
123
|
-
success: (message: string) => {
|
|
124
|
-
const d = getActiveDisplay();
|
|
125
|
-
if (d) return d.subEvent(`\x1b[32m✔\x1b[0m ${message}`);
|
|
126
|
-
p.log.success(message);
|
|
127
|
-
},
|
|
128
|
-
error: (message: string) => {
|
|
129
|
-
const d = getActiveDisplay();
|
|
130
|
-
if (d) return d.subEvent(`\x1b[31m✗\x1b[0m ${message}`);
|
|
131
|
-
p.log.error(message);
|
|
132
|
-
},
|
|
133
|
-
warn: (message: string) => {
|
|
134
|
-
const d = getActiveDisplay();
|
|
135
|
-
if (d) return d.subEvent(`\x1b[33m⚠\x1b[0m ${message}`);
|
|
136
|
-
p.log.warn(message);
|
|
137
|
-
},
|
|
138
|
-
info: (message: string) => {
|
|
139
|
-
const d = getActiveDisplay();
|
|
140
|
-
if (d) return d.instantEvent(message);
|
|
141
|
-
p.log.info(message);
|
|
142
|
-
},
|
|
143
|
-
message: (message: string) => {
|
|
144
|
-
const d = getActiveDisplay();
|
|
145
|
-
if (d) return d.subEvent(message);
|
|
146
|
-
p.log.message(message);
|
|
147
|
-
},
|
|
148
|
-
};
|
|
@@ -8,10 +8,10 @@
|
|
|
8
8
|
* secrets therefore travel by **flag or env var**, never over the event bus.
|
|
9
9
|
*
|
|
10
10
|
* The resolution order is: explicit `--<flag>` → `$ENV` → (only when stdin is
|
|
11
|
-
* a TTY) a local
|
|
11
|
+
* a TTY) a local masked `password` prompt → otherwise a fail-fast error naming
|
|
12
12
|
* the flag and env var. This keeps a zero-TTY `service add` possible while the
|
|
13
13
|
* credential never lands on the bus, and the local password prompt is the one
|
|
14
|
-
*
|
|
14
|
+
* direct prompt the recurrence gate (D6) tolerates — precisely because a
|
|
15
15
|
* flag/env path always exists alongside it.
|
|
16
16
|
*/
|
|
17
17
|
|
|
@@ -29,7 +29,7 @@ export interface ServiceCredentialSpec {
|
|
|
29
29
|
}
|
|
30
30
|
|
|
31
31
|
/**
|
|
32
|
-
* Resolve a service-credential secret from flag → env → (TTY)
|
|
32
|
+
* Resolve a service-credential secret from flag → env → (TTY) password prompt →
|
|
33
33
|
* error. Returns the secret value as a string. Throws an actionable error when
|
|
34
34
|
* no value is available on a non-TTY run.
|
|
35
35
|
*/
|
|
@@ -0,0 +1,94 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Recurrence gate for celilo#699 — ordinary stdout carries no decoration, and
|
|
3
|
+
* diagnostics do not share the stream with results.
|
|
4
|
+
*
|
|
5
|
+
* The CLI used to render every successful message through `@clack/prompts`,
|
|
6
|
+
* which prefixed each line with `│ ` and coloured it. Two costs followed, and
|
|
7
|
+
* this file pins both shut:
|
|
8
|
+
*
|
|
9
|
+
* 1. `line.startsWith('<module-id> ')` over `celilo module list` matched
|
|
10
|
+
* nothing, because the line actually began `│ caddy (v2.2.0) - …`. In
|
|
11
|
+
* celilo#695 that read as a MISSING MODULE rather than as a parse failure,
|
|
12
|
+
* and cost a full e2e run to find. `e2e/tests/module-pause.test.ts` asserts
|
|
13
|
+
* the same property against the real fleet topology; this is its fast
|
|
14
|
+
* equivalent, so a regression is caught in seconds rather than in Docker.
|
|
15
|
+
*
|
|
16
|
+
* 2. clack wrote errors to stdout, so no caller could tell a result from a
|
|
17
|
+
* complaint about producing one — `src/test-utils/cli.ts` carried a comment
|
|
18
|
+
* saying exactly that, twice, and merged both streams to cope.
|
|
19
|
+
*
|
|
20
|
+
* Spawns the real CLI rather than using `CLIContext`: that harness runs the CLI
|
|
21
|
+
* in `CLI_SERVER_MODE`, which returns `result.message` over a protocol and
|
|
22
|
+
* never reaches the stdout writer under test here.
|
|
23
|
+
*/
|
|
24
|
+
|
|
25
|
+
import { afterAll, beforeAll, describe, expect, test } from 'bun:test';
|
|
26
|
+
import { spawnSync } from 'node:child_process';
|
|
27
|
+
import { type IntegrationTestContext, setupIntegrationTest } from '@/test-utils/integration';
|
|
28
|
+
|
|
29
|
+
let ctx: IntegrationTestContext;
|
|
30
|
+
|
|
31
|
+
function celilo(command: string): { stdout: string; stderr: string; status: number | null } {
|
|
32
|
+
const result = spawnSync('bun', ['run', 'src/cli/index.ts', ...command.split(' ')], {
|
|
33
|
+
encoding: 'utf8',
|
|
34
|
+
env: {
|
|
35
|
+
...process.env,
|
|
36
|
+
CELILO_DB_PATH: ctx.dbPath,
|
|
37
|
+
CELILO_DATA_DIR: ctx.dataDir,
|
|
38
|
+
CELILO_SUPPRESS_DEPRECATION: '1',
|
|
39
|
+
},
|
|
40
|
+
stdio: ['ignore', 'pipe', 'pipe'],
|
|
41
|
+
timeout: 60_000,
|
|
42
|
+
});
|
|
43
|
+
return { stdout: result.stdout, stderr: result.stderr, status: result.status };
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
/** The `│` clack used to open every rendered line with. */
|
|
47
|
+
const BOX_DRAWING = /[│┌└├─]/;
|
|
48
|
+
const ANSI = /\x1b\[[0-9;]*m/;
|
|
49
|
+
|
|
50
|
+
describe('celilo#699 — stdout is undecorated', () => {
|
|
51
|
+
beforeAll(async () => {
|
|
52
|
+
ctx = await setupIntegrationTest();
|
|
53
|
+
// A module has to exist for `module list` to print a roster line at all —
|
|
54
|
+
// an empty roster would pass every assertion below without testing them.
|
|
55
|
+
const imported = celilo('module import ../../modules/celilo-mgmt');
|
|
56
|
+
expect(imported.status, `module import failed:\n${imported.stderr}`).toBe(0);
|
|
57
|
+
});
|
|
58
|
+
|
|
59
|
+
afterAll(async () => {
|
|
60
|
+
await ctx.cleanup();
|
|
61
|
+
});
|
|
62
|
+
|
|
63
|
+
test('module list lines start with the module id, with no preprocessing', () => {
|
|
64
|
+
const { stdout, status, stderr } = celilo('module list');
|
|
65
|
+
expect(status, `module list failed:\n${stderr}`).toBe(0);
|
|
66
|
+
|
|
67
|
+
// The exact shape celilo#695 tried and failed to match. No ANSI stripping,
|
|
68
|
+
// no prefix trimming — if this needs either, the bug is back.
|
|
69
|
+
const line = stdout.split('\n').find((l) => l.startsWith('celilo-mgmt '));
|
|
70
|
+
|
|
71
|
+
expect(
|
|
72
|
+
line,
|
|
73
|
+
`No line began with "celilo-mgmt ". stdout was:\n${JSON.stringify(stdout)}`,
|
|
74
|
+
).toBeDefined();
|
|
75
|
+
});
|
|
76
|
+
|
|
77
|
+
test('module list stdout carries no box-drawing or ANSI', () => {
|
|
78
|
+
const { stdout } = celilo('module list');
|
|
79
|
+
expect(BOX_DRAWING.test(stdout), `box-drawing in stdout:\n${JSON.stringify(stdout)}`).toBe(
|
|
80
|
+
false,
|
|
81
|
+
);
|
|
82
|
+
expect(ANSI.test(stdout), `ANSI in stdout:\n${JSON.stringify(stdout)}`).toBe(false);
|
|
83
|
+
});
|
|
84
|
+
|
|
85
|
+
test('a failing command writes its diagnostic to stderr, not stdout', () => {
|
|
86
|
+
const { stdout, stderr, status } = celilo('module where no-such-module-exists');
|
|
87
|
+
|
|
88
|
+
expect(status).not.toBe(0);
|
|
89
|
+
expect(stderr).toContain('Error');
|
|
90
|
+
// The whole point: stdout stays empty so a caller parsing it is not handed
|
|
91
|
+
// an error message where a result belongs.
|
|
92
|
+
expect(stdout.trim(), `error text leaked to stdout: ${JSON.stringify(stdout)}`).toBe('');
|
|
93
|
+
});
|
|
94
|
+
});
|
package/src/cli/types.ts
CHANGED
|
@@ -20,8 +20,13 @@ export interface CommandSuccess {
|
|
|
20
20
|
message: string;
|
|
21
21
|
data?: unknown;
|
|
22
22
|
/**
|
|
23
|
-
*
|
|
24
|
-
*
|
|
23
|
+
* Marks a command whose `message` is a machine-readable payload (JSON, a
|
|
24
|
+
* unit file, a secret) rather than prose for a human.
|
|
25
|
+
*
|
|
26
|
+
* Every successful message now reaches stdout verbatim, so this no longer
|
|
27
|
+
* selects a different destination. It is kept as the contract: these are the
|
|
28
|
+
* commands a script parses, and nothing may ever decorate them (celilo#698,
|
|
29
|
+
* where ten JSON payloads were rendered unparseable by the formatter).
|
|
25
30
|
*/
|
|
26
31
|
rawOutput?: boolean;
|
|
27
32
|
}
|
package/src/db/schema.ts
CHANGED
|
@@ -11,7 +11,17 @@ import {
|
|
|
11
11
|
|
|
12
12
|
/**
|
|
13
13
|
* Module lifecycle states
|
|
14
|
-
* IMPORTED, VALIDATED, CONFIGURED, GENERATING, ERROR, DEPLOYING, INSTALLED, VERIFIED, UNINSTALLING
|
|
14
|
+
* IMPORTED, VALIDATED, CONFIGURED, GENERATING, ERROR, DEPLOYING, INSTALLED, VERIFIED, UNINSTALLING, PAUSED
|
|
15
|
+
*
|
|
16
|
+
* `PAUSED` is a real member of this union rather than a side flag, and that is
|
|
17
|
+
* the point (openspec/changes/module-pause-lifecycle/design.md D1): adding it
|
|
18
|
+
* makes the type-checker enumerate every site that must now consider
|
|
19
|
+
* paused-ness. A `pausedAt`-only flag would leave every `state === 'VERIFIED'`
|
|
20
|
+
* comparison silently compiling while quietly reading a paused module as live.
|
|
21
|
+
*
|
|
22
|
+
* There is deliberately no `prePauseState`: pause is legal only from a settled
|
|
23
|
+
* state and unpause redeploys, so the deploy path decides the resulting state
|
|
24
|
+
* and there is nothing to restore.
|
|
15
25
|
*/
|
|
16
26
|
export type ModuleState =
|
|
17
27
|
| 'IMPORTED'
|
|
@@ -22,23 +32,65 @@ export type ModuleState =
|
|
|
22
32
|
| 'INSTALLED'
|
|
23
33
|
| 'VERIFIED'
|
|
24
34
|
| 'ERROR'
|
|
25
|
-
| 'UNINSTALLING'
|
|
35
|
+
| 'UNINSTALLING'
|
|
36
|
+
| 'PAUSED';
|
|
37
|
+
|
|
38
|
+
/**
|
|
39
|
+
* States a module may be paused FROM (design D1). `ERROR` is deliberately
|
|
40
|
+
* included: quiescing a broken module to stop alert noise while working on it
|
|
41
|
+
* is legitimate, and refusing would push the operator toward silencing those
|
|
42
|
+
* alerts by some less visible route.
|
|
43
|
+
*/
|
|
44
|
+
export const PAUSABLE_STATES = [
|
|
45
|
+
'INSTALLED',
|
|
46
|
+
'VERIFIED',
|
|
47
|
+
'ERROR',
|
|
48
|
+
] as const satisfies readonly ModuleState[];
|
|
49
|
+
|
|
50
|
+
/**
|
|
51
|
+
* States that mean "a lifecycle transition is under way". Pausing one of these
|
|
52
|
+
* would strand the transition, so pause is refused with a distinct message from
|
|
53
|
+
* the never-deployed case.
|
|
54
|
+
*/
|
|
55
|
+
export const IN_FLIGHT_STATES = [
|
|
56
|
+
'GENERATING',
|
|
57
|
+
'DEPLOYING',
|
|
58
|
+
'UNINSTALLING',
|
|
59
|
+
] as const satisfies readonly ModuleState[];
|
|
26
60
|
|
|
27
61
|
/**
|
|
28
62
|
* Modules table - stores module metadata and manifest data
|
|
29
63
|
*/
|
|
30
|
-
export const modules = sqliteTable(
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
64
|
+
export const modules = sqliteTable(
|
|
65
|
+
'modules',
|
|
66
|
+
{
|
|
67
|
+
id: text('id').primaryKey(),
|
|
68
|
+
name: text('name').notNull(),
|
|
69
|
+
version: text('version').notNull(),
|
|
70
|
+
description: text('description'),
|
|
71
|
+
state: text('state').$type<ModuleState>().notNull().default('IMPORTED'),
|
|
72
|
+
manifestData: text('manifest_data', { mode: 'json' })
|
|
73
|
+
.$type<Record<string, unknown>>()
|
|
74
|
+
.notNull(),
|
|
75
|
+
sourcePath: text('source_path').notNull(),
|
|
76
|
+
importedAt: integer('imported_at', { mode: 'timestamp' }).notNull().default(sql`(unixepoch())`),
|
|
77
|
+
updatedAt: integer('updated_at', { mode: 'timestamp' }).notNull().default(sql`(unixepoch())`),
|
|
78
|
+
errorMessage: text('error_message'),
|
|
79
|
+
/**
|
|
80
|
+
* When the module was paused. Null unless `state = 'PAUSED'`. The state alone
|
|
81
|
+
* cannot answer "how long", and the DURATION is what makes a forgotten pause
|
|
82
|
+
* detectable (design D7) — every paused-module report carries the age.
|
|
83
|
+
*/
|
|
84
|
+
pausedAt: integer('paused_at', { mode: 'timestamp' }),
|
|
85
|
+
/** Operator-supplied explanation, so the row explains itself. */
|
|
86
|
+
pauseReason: text('pause_reason'),
|
|
87
|
+
},
|
|
88
|
+
(table) => ({
|
|
89
|
+
// Every management-API response asks "is anything paused?" (design D7), so
|
|
90
|
+
// that lookup must stay a single indexed hit rather than a table scan.
|
|
91
|
+
stateIdx: index('modules_state_idx').on(table.state),
|
|
92
|
+
}),
|
|
93
|
+
);
|
|
42
94
|
|
|
43
95
|
/**
|
|
44
96
|
* Module configuration - user-provided key-value pairs.
|
|
@@ -807,7 +859,13 @@ export const backups = sqliteTable('backups', {
|
|
|
807
859
|
* pid is no longer alive is treated as abandoned (the process crashed before
|
|
808
860
|
* the completion update landed) and ignored by in-flight checks.
|
|
809
861
|
*/
|
|
810
|
-
export type ModuleOperationKind =
|
|
862
|
+
export type ModuleOperationKind =
|
|
863
|
+
| 'deploy'
|
|
864
|
+
| 'uninstall'
|
|
865
|
+
| 'backup'
|
|
866
|
+
| 'restore'
|
|
867
|
+
| 'pause'
|
|
868
|
+
| 'unpause';
|
|
811
869
|
export type ModuleOperationStatus = 'in_progress' | 'completed' | 'failed';
|
|
812
870
|
|
|
813
871
|
export const moduleOperations = sqliteTable('module_operations', {
|
|
@@ -49,6 +49,13 @@ export interface RunNamedHookResult extends HookResult {
|
|
|
49
49
|
* gracefully when a module has no `on_uninstall` defined.
|
|
50
50
|
*/
|
|
51
51
|
notDefined?: boolean;
|
|
52
|
+
/**
|
|
53
|
+
* True when the hook was not run because the module is PAUSED. Reported as
|
|
54
|
+
* success rather than failure: the module is deliberately quiesced, and a
|
|
55
|
+
* failure here would be retried by the bus and then alerted on — paging the
|
|
56
|
+
* operator about the pause they took themselves.
|
|
57
|
+
*/
|
|
58
|
+
skippedPaused?: boolean;
|
|
52
59
|
}
|
|
53
60
|
|
|
54
61
|
/**
|
|
@@ -82,6 +89,27 @@ export async function runNamedHook(
|
|
|
82
89
|
};
|
|
83
90
|
}
|
|
84
91
|
|
|
92
|
+
// Quiescence for a paused module (openspec/changes/module-pause-lifecycle,
|
|
93
|
+
// tasks 2.1/2.2). This is the chokepoint every non-lifecycle invocation
|
|
94
|
+
// funnels through — bus dispatch, timer fan-out, aspect fan-out,
|
|
95
|
+
// public-web republish, the dns-provider backfill, and `module run-hook` —
|
|
96
|
+
// so guarding here covers the paths individually rather than each caller
|
|
97
|
+
// remembering to.
|
|
98
|
+
//
|
|
99
|
+
// The exemptions are decided from the hook NAME, not a caller-supplied flag
|
|
100
|
+
// (Rule 10.3): `on_install` is how unpause redeploys the module back to life,
|
|
101
|
+
// and `on_uninstall` is how a paused module is removed — which is the entire
|
|
102
|
+
// point of pausing it. Both must run while `state` is still PAUSED.
|
|
103
|
+
const LIFECYCLE_HOOKS: readonly HookName[] = ['on_install', 'on_uninstall'];
|
|
104
|
+
if (module.state === 'PAUSED' && !LIFECYCLE_HOOKS.includes(hookName)) {
|
|
105
|
+
return {
|
|
106
|
+
success: true,
|
|
107
|
+
outputs: {},
|
|
108
|
+
duration: Date.now() - startedAt,
|
|
109
|
+
skippedPaused: true,
|
|
110
|
+
};
|
|
111
|
+
}
|
|
112
|
+
|
|
85
113
|
const manifest = module.manifestData as ModuleManifest;
|
|
86
114
|
const hookDef = manifest.hooks?.[hookName as keyof typeof manifest.hooks];
|
|
87
115
|
if (!hookDef) {
|
|
@@ -35,6 +35,7 @@ const TOPOLOGY: SuppressionTopology = {
|
|
|
35
35
|
const noSuppression = {
|
|
36
36
|
suppressible: true,
|
|
37
37
|
modulesInDeployWindow: new Set<string>(),
|
|
38
|
+
pausedModules: new Set<string>(),
|
|
38
39
|
topology: TOPOLOGY,
|
|
39
40
|
};
|
|
40
41
|
|
|
@@ -185,6 +186,7 @@ describe('findSuppressor — guards', () => {
|
|
|
185
186
|
firingKeys,
|
|
186
187
|
suppressible: false,
|
|
187
188
|
modulesInDeployWindow: new Set(),
|
|
189
|
+
pausedModules: new Set(),
|
|
188
190
|
topology: TOPOLOGY,
|
|
189
191
|
}),
|
|
190
192
|
).toBeNull();
|
|
@@ -197,6 +199,7 @@ describe('findSuppressor — guards', () => {
|
|
|
197
199
|
firingKeys: new Set(),
|
|
198
200
|
suppressible: true,
|
|
199
201
|
modulesInDeployWindow: new Set(['forgejo']),
|
|
202
|
+
pausedModules: new Set(),
|
|
200
203
|
topology: TOPOLOGY,
|
|
201
204
|
}),
|
|
202
205
|
).toEqual({ kind: 'deploy_window', moduleId: 'forgejo' });
|
|
@@ -209,6 +212,7 @@ describe('findSuppressor — guards', () => {
|
|
|
209
212
|
firingKeys: new Set(),
|
|
210
213
|
suppressible: true,
|
|
211
214
|
modulesInDeployWindow: new Set(['forgejo']),
|
|
215
|
+
pausedModules: new Set(),
|
|
212
216
|
topology: TOPOLOGY,
|
|
213
217
|
}),
|
|
214
218
|
).toBeNull();
|
|
@@ -221,6 +225,7 @@ describe('findSuppressor — guards', () => {
|
|
|
221
225
|
firingKeys: new Set([machineAlertKey('iot')]),
|
|
222
226
|
suppressible: true,
|
|
223
227
|
modulesInDeployWindow: new Set(['homebridge']),
|
|
228
|
+
pausedModules: new Set(),
|
|
224
229
|
topology: TOPOLOGY,
|
|
225
230
|
}),
|
|
226
231
|
).toEqual({ kind: 'deploy_window', moduleId: 'homebridge' });
|
|
@@ -106,12 +106,20 @@ export interface SuppressorLookup {
|
|
|
106
106
|
suppressible: boolean;
|
|
107
107
|
/** Modules currently inside a deploy window. */
|
|
108
108
|
modulesInDeployWindow: ReadonlySet<string>;
|
|
109
|
+
/**
|
|
110
|
+
* Modules currently PAUSED. A pause is a deliberate quiescing, so its alerts
|
|
111
|
+
* are explained by the pause itself (openspec/changes/module-pause-lifecycle,
|
|
112
|
+
* task 2.3) — same mechanism as a deploy window, with the pause as the source
|
|
113
|
+
* instead of an ancestor alert.
|
|
114
|
+
*/
|
|
115
|
+
pausedModules: ReadonlySet<string>;
|
|
109
116
|
topology: SuppressionTopology;
|
|
110
117
|
}
|
|
111
118
|
|
|
112
119
|
export type Suppressor =
|
|
113
120
|
| { kind: 'alert'; key: string }
|
|
114
|
-
| { kind: 'deploy_window'; moduleId: string }
|
|
121
|
+
| { kind: 'deploy_window'; moduleId: string }
|
|
122
|
+
| { kind: 'paused'; moduleId: string };
|
|
115
123
|
|
|
116
124
|
/**
|
|
117
125
|
* Find what is suppressing `key`, or null if it should be reported.
|
|
@@ -128,6 +136,15 @@ export function findSuppressor(lookup: SuppressorLookup): Suppressor | null {
|
|
|
128
136
|
|
|
129
137
|
const parsed = parseAlertKey(lookup.key);
|
|
130
138
|
|
|
139
|
+
// A pause is checked before a deploy window because it is the longer-lived
|
|
140
|
+
// and more consequential explanation: a paused module may also be inside a
|
|
141
|
+
// deploy window (unpause redeploys), and "paused" is the fact the operator
|
|
142
|
+
// needs to see. Attributed to the pause rather than suppressed anonymously —
|
|
143
|
+
// silently dropping the alert is what turns a pause into an invisible outage.
|
|
144
|
+
if (parsed?.source === 'module' && lookup.pausedModules.has(parsed.moduleId)) {
|
|
145
|
+
return { kind: 'paused', moduleId: parsed.moduleId };
|
|
146
|
+
}
|
|
147
|
+
|
|
131
148
|
// A deploy is the same mechanism with a window as the source instead of an
|
|
132
149
|
// ancestor alert — which is why deploy auto-silencing is not a second feature.
|
|
133
150
|
if (parsed?.source === 'module' && lookup.modulesInDeployWindow.has(parsed.moduleId)) {
|
|
@@ -50,6 +50,7 @@ describe('runSweep', () => {
|
|
|
50
50
|
monitorDeps: monitorDeps(result, now),
|
|
51
51
|
loadTopology: () => TOPOLOGY,
|
|
52
52
|
loadDeployWindowModules: () => new Set(),
|
|
53
|
+
loadPausedModules: () => new Set(),
|
|
53
54
|
isSuppressible: () => true,
|
|
54
55
|
// No routes configured: the sweep must still run everything else.
|
|
55
56
|
notifyDepsFor: () => null,
|
|
@@ -43,6 +43,8 @@ export interface SweepDeps {
|
|
|
43
43
|
loadTopology(): SuppressionTopology;
|
|
44
44
|
/** Modules currently inside a deploy window. */
|
|
45
45
|
loadDeployWindowModules(): Set<string>;
|
|
46
|
+
/** Ids of modules currently PAUSED — a pause explains its own module's alerts. */
|
|
47
|
+
loadPausedModules(): Set<string>;
|
|
46
48
|
/** Whether the monitor owning an alert may be suppressed at all. */
|
|
47
49
|
isSuppressible(alert: Alert): boolean;
|
|
48
50
|
/** Compose the per-alert notification context. Null when nothing can page. */
|
|
@@ -156,6 +158,7 @@ export async function runSweep(
|
|
|
156
158
|
);
|
|
157
159
|
const topology = deps.loadTopology();
|
|
158
160
|
const deployWindows = deps.loadDeployWindowModules();
|
|
161
|
+
const paused = deps.loadPausedModules();
|
|
159
162
|
|
|
160
163
|
for (const alert of live) {
|
|
161
164
|
const suppressor = findSuppressor({
|
|
@@ -163,13 +166,20 @@ export async function runSweep(
|
|
|
163
166
|
firingKeys,
|
|
164
167
|
suppressible: deps.isSuppressible(alert),
|
|
165
168
|
modulesInDeployWindow: deployWindows,
|
|
169
|
+
pausedModules: paused,
|
|
166
170
|
topology,
|
|
167
171
|
});
|
|
168
172
|
const wasSuppressed = alert.state === 'suppressed';
|
|
169
173
|
if (suppressor && !wasSuppressed) {
|
|
170
174
|
markSuppressed(db, alert.id, {
|
|
171
175
|
alertId: suppressor.kind === 'alert' ? suppressor.key : undefined,
|
|
172
|
-
|
|
176
|
+
// A pause is recorded on the same column as a deploy window: both are
|
|
177
|
+
// "a module-scoped condition explains this", and the operator reads the
|
|
178
|
+
// module id either way.
|
|
179
|
+
windowId:
|
|
180
|
+
suppressor.kind === 'deploy_window' || suppressor.kind === 'paused'
|
|
181
|
+
? suppressor.moduleId
|
|
182
|
+
: undefined,
|
|
173
183
|
});
|
|
174
184
|
report.suppressed++;
|
|
175
185
|
} else if (!suppressor && wasSuppressed) {
|
|
@@ -314,7 +314,7 @@ export async function busInterviewGuarded<TReply>(
|
|
|
314
314
|
* Ask a single generic interview question over the bus and return the
|
|
315
315
|
* responder's answer (ISS-0127). The generic counterpart to the deploy's
|
|
316
316
|
* config/secret/ensure interview: any operator command can call this to make
|
|
317
|
-
* its prompts headlessly drivable instead of
|
|
317
|
+
* its prompts headlessly drivable instead of prompting on stdin directly.
|
|
318
318
|
*
|
|
319
319
|
* The return type is `unknown` because the runtime shape depends on
|
|
320
320
|
* `payload.kind`; prefer the typed wrappers (`askText`, `askSelect`,
|
|
@@ -345,7 +345,7 @@ export async function askInterview(
|
|
|
345
345
|
*
|
|
346
346
|
* This is the shared lifecycle every migrated command wraps its interview in,
|
|
347
347
|
* so the start/close boilerplate lives in exactly one place. The dynamic
|
|
348
|
-
* import keeps
|
|
348
|
+
* import keeps the terminal renderer out of the non-TTY path's module graph.
|
|
349
349
|
*/
|
|
350
350
|
export async function withInterviewSession<T>(fn: () => Promise<T>): Promise<T> {
|
|
351
351
|
const responder = process.stdin.isTTY
|
|
@@ -3,7 +3,7 @@
|
|
|
3
3
|
*
|
|
4
4
|
* Drives `interviewForMissingSecrets` against a real sqlite bus + real
|
|
5
5
|
* encrypted store + a programmatic test responder. No fixture modules,
|
|
6
|
-
* no machines, no
|
|
6
|
+
* no machines, no terminal — the responder is just a `bus.watch` that
|
|
7
7
|
* mimics what `terminal-responder.ts` does.
|
|
8
8
|
*
|
|
9
9
|
* Covers what stage 3 introduced:
|
|
@@ -40,8 +40,20 @@ export interface DnsRegistrationRow {
|
|
|
40
40
|
* triggers for FK-cascaded deletes only when `recursive_triggers` is on,
|
|
41
41
|
* and reads are the only thing that consumes the ledger. Move it into a
|
|
42
42
|
* trigger if something ever reads these rows without going through here.
|
|
43
|
+
*
|
|
44
|
+
* It CHECKS before deleting, so the overwhelmingly common case (nothing
|
|
45
|
+
* orphaned) stays a pure read. The first version ran the DELETE
|
|
46
|
+
* unconditionally, which took a write lock on every list — including the
|
|
47
|
+
* refresh hook's, and `celilo dns registrations`. A read that quietly writes
|
|
48
|
+
* is a surprise on its own, and on SQLite it is a surprise that serialises
|
|
49
|
+
* against every other writer for no benefit.
|
|
43
50
|
*/
|
|
44
51
|
function pruneOrphanedRegistrations(db: DbClient): void {
|
|
52
|
+
const orphaned = sql`SELECT 1 FROM dns_registrations WHERE NOT EXISTS (
|
|
53
|
+
SELECT 1 FROM dns_registration_consumers c WHERE c.registration_id = dns_registrations.id
|
|
54
|
+
) LIMIT 1`;
|
|
55
|
+
if (!db.get(orphaned)) return;
|
|
56
|
+
|
|
45
57
|
db.run(
|
|
46
58
|
sql`DELETE FROM dns_registrations WHERE NOT EXISTS (
|
|
47
59
|
SELECT 1 FROM dns_registration_consumers c WHERE c.registration_id = dns_registrations.id
|