@celilo/cli 0.22.0 → 0.24.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CELILO_CORE_MODULES.md +2 -2
- package/CELILO_SUBSYSTEMS.md +61 -9
- package/drizzle/0024_module_pause.sql +20 -0
- package/drizzle/meta/_journal.json +8 -1
- package/package.json +7 -7
- package/src/__integration__/container-services-cli.integration.test.ts +8 -2
- package/src/api/remote-client.test.ts +6 -5
- package/src/api/serve.ts +41 -7
- package/src/api-clients/proxmox.ts +34 -0
- package/src/cli/commands/alerts-act.ts +1 -1
- package/src/cli/commands/alerts-sweep.ts +2 -0
- package/src/cli/commands/backup-create.ts +26 -11
- package/src/cli/commands/backup-list.test.ts +83 -0
- package/src/cli/commands/backup-list.ts +67 -3
- package/src/cli/commands/backup-prune.ts +17 -17
- package/src/cli/commands/backup-sweep.ts +20 -8
- package/src/cli/commands/events.ts +34 -3
- package/src/cli/commands/firewall-interface-list.test.ts +85 -0
- package/src/cli/commands/firewall-interface-list.ts +123 -0
- package/src/cli/commands/machine-add.ts +30 -2
- package/src/cli/commands/module-config.test.ts +64 -2
- package/src/cli/commands/module-config.ts +159 -8
- package/src/cli/commands/module-deploy.ts +2 -2
- package/src/cli/commands/module-health.ts +1 -0
- package/src/cli/commands/module-import.ts +3 -3
- package/src/cli/commands/module-list.ts +12 -1
- package/src/cli/commands/module-pause.ts +317 -0
- package/src/cli/commands/module-remove.ts +78 -40
- package/src/cli/commands/module-status.ts +127 -4
- package/src/cli/commands/module-update.test.ts +1 -1
- package/src/cli/commands/monitor.ts +116 -19
- package/src/cli/commands/proxmox-template-selection.ts +1 -1
- package/src/cli/commands/status.ts +25 -3
- package/src/cli/commands/system-migrate.ts +14 -0
- package/src/cli/commands/system-update.ts +4 -1
- package/src/cli/completion.ts +39 -9
- package/src/cli/fuel-gauge.ts +4 -4
- package/src/cli/index.ts +104 -22
- package/src/cli/json-output.test.ts +162 -0
- package/src/cli/prompts.ts +53 -74
- package/src/cli/service-credential.ts +3 -3
- package/src/cli/stdout-is-undecorated.test.ts +94 -0
- package/src/cli/tui/audit-state.ts +2 -0
- package/src/cli/types.ts +7 -2
- package/src/db/schema.ts +73 -15
- package/src/hooks/capability-loader.ts +130 -4
- package/src/hooks/run-named-hook.ts +28 -0
- package/src/hooks/types.ts +2 -1
- package/src/manifest/contracts/v1.ts +16 -0
- package/src/manifest/schema.ts +40 -65
- package/src/services/alerting/builtin-monitors.test.ts +18 -10
- package/src/services/alerting/cadence-migration.test.ts +155 -0
- package/src/services/alerting/cadence-migration.ts +90 -0
- package/src/services/alerting/coverage-source.ts +8 -11
- package/src/services/alerting/deploy-hooks.test.ts +16 -7
- package/src/services/alerting/deploy-hooks.ts +11 -5
- package/src/services/alerting/health-cadence.test.ts +58 -0
- package/src/services/alerting/health-cadence.ts +128 -0
- package/src/services/alerting/health-coverage.ts +18 -8
- package/src/services/alerting/monitors.ts +50 -15
- package/src/services/alerting/suppression.test.ts +5 -0
- package/src/services/alerting/suppression.ts +18 -1
- package/src/services/alerting/sweep-runner.test.ts +52 -3
- package/src/services/alerting/sweep-runner.ts +41 -8
- package/src/services/audit/backup-source.ts +24 -1
- package/src/services/audit/backups.test.ts +95 -10
- package/src/services/audit/backups.ts +40 -37
- package/src/services/audit/interface-classification.test.ts +220 -0
- package/src/services/audit/interface-classification.ts +167 -0
- package/src/services/audit/types.ts +2 -1
- package/src/services/backup-age-agreement.test.ts +118 -0
- package/src/services/backup-create.ts +36 -30
- package/src/services/backup-metadata.ts +52 -1
- package/src/services/backup-retention.test.ts +123 -0
- package/src/services/backup-retention.ts +66 -5
- package/src/services/backup-schedule.test.ts +166 -0
- package/src/services/backup-schedule.ts +105 -15
- package/src/services/backup-staging.ts +14 -1
- package/src/services/backup-sweep.test.ts +22 -3
- package/src/services/backup-sweep.ts +15 -5
- package/src/services/bus-interview.ts +2 -2
- package/src/services/bus-secret-flow.test.ts +1 -1
- package/src/services/cadence.test.ts +97 -0
- package/src/services/cadence.ts +165 -0
- package/src/services/fleet-checks.ts +48 -0
- package/src/services/machine-detector.ts +23 -1
- package/src/services/module-config.ts +33 -0
- package/src/services/module-deploy.ts +1 -1
- package/src/services/module-pause-observability.test.ts +224 -0
- package/src/services/module-pause-quiescence.test.ts +163 -0
- package/src/services/module-pause.test.ts +573 -0
- package/src/services/module-pause.ts +544 -0
- package/src/services/remove-guard.test.ts +175 -0
- package/src/services/remove-guard.ts +109 -0
- package/src/services/storage-providers/s3.test.ts +96 -13
- package/src/services/storage-providers/s3.ts +48 -15
- package/src/services/terminal-responder.ts +16 -16
- package/src/services/update/dep-graph.test.ts +33 -4
- package/src/services/update/dep-graph.ts +39 -17
- package/src/services/zone-detector.test.ts +34 -3
- package/src/services/zone-detector.ts +32 -49
- package/src/test-utils/cli.ts +15 -14
- package/src/test-utils/integration-guard.ts +26 -0
- package/src/test-utils/setup-test-db.ts +13 -23
|
@@ -0,0 +1,94 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Recurrence gate for celilo#699 — ordinary stdout carries no decoration, and
|
|
3
|
+
* diagnostics do not share the stream with results.
|
|
4
|
+
*
|
|
5
|
+
* The CLI used to render every successful message through `@clack/prompts`,
|
|
6
|
+
* which prefixed each line with `│ ` and coloured it. Two costs followed, and
|
|
7
|
+
* this file pins both shut:
|
|
8
|
+
*
|
|
9
|
+
* 1. `line.startsWith('<module-id> ')` over `celilo module list` matched
|
|
10
|
+
* nothing, because the line actually began `│ caddy (v2.2.0) - …`. In
|
|
11
|
+
* celilo#695 that read as a MISSING MODULE rather than as a parse failure,
|
|
12
|
+
* and cost a full e2e run to find. `e2e/tests/module-pause.test.ts` asserts
|
|
13
|
+
* the same property against the real fleet topology; this is its fast
|
|
14
|
+
* equivalent, so a regression is caught in seconds rather than in Docker.
|
|
15
|
+
*
|
|
16
|
+
* 2. clack wrote errors to stdout, so no caller could tell a result from a
|
|
17
|
+
* complaint about producing one — `src/test-utils/cli.ts` carried a comment
|
|
18
|
+
* saying exactly that, twice, and merged both streams to cope.
|
|
19
|
+
*
|
|
20
|
+
* Spawns the real CLI rather than using `CLIContext`: that harness runs the CLI
|
|
21
|
+
* in `CLI_SERVER_MODE`, which returns `result.message` over a protocol and
|
|
22
|
+
* never reaches the stdout writer under test here.
|
|
23
|
+
*/
|
|
24
|
+
|
|
25
|
+
import { afterAll, beforeAll, describe, expect, test } from 'bun:test';
|
|
26
|
+
import { spawnSync } from 'node:child_process';
|
|
27
|
+
import { type IntegrationTestContext, setupIntegrationTest } from '@/test-utils/integration';
|
|
28
|
+
|
|
29
|
+
let ctx: IntegrationTestContext;
|
|
30
|
+
|
|
31
|
+
function celilo(command: string): { stdout: string; stderr: string; status: number | null } {
|
|
32
|
+
const result = spawnSync('bun', ['run', 'src/cli/index.ts', ...command.split(' ')], {
|
|
33
|
+
encoding: 'utf8',
|
|
34
|
+
env: {
|
|
35
|
+
...process.env,
|
|
36
|
+
CELILO_DB_PATH: ctx.dbPath,
|
|
37
|
+
CELILO_DATA_DIR: ctx.dataDir,
|
|
38
|
+
CELILO_SUPPRESS_DEPRECATION: '1',
|
|
39
|
+
},
|
|
40
|
+
stdio: ['ignore', 'pipe', 'pipe'],
|
|
41
|
+
timeout: 60_000,
|
|
42
|
+
});
|
|
43
|
+
return { stdout: result.stdout, stderr: result.stderr, status: result.status };
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
/** The `│` clack used to open every rendered line with. */
|
|
47
|
+
const BOX_DRAWING = /[│┌└├─]/;
|
|
48
|
+
const ANSI = /\x1b\[[0-9;]*m/;
|
|
49
|
+
|
|
50
|
+
describe('celilo#699 — stdout is undecorated', () => {
|
|
51
|
+
beforeAll(async () => {
|
|
52
|
+
ctx = await setupIntegrationTest();
|
|
53
|
+
// A module has to exist for `module list` to print a roster line at all —
|
|
54
|
+
// an empty roster would pass every assertion below without testing them.
|
|
55
|
+
const imported = celilo('module import ../../modules/celilo-mgmt');
|
|
56
|
+
expect(imported.status, `module import failed:\n${imported.stderr}`).toBe(0);
|
|
57
|
+
});
|
|
58
|
+
|
|
59
|
+
afterAll(async () => {
|
|
60
|
+
await ctx.cleanup();
|
|
61
|
+
});
|
|
62
|
+
|
|
63
|
+
test('module list lines start with the module id, with no preprocessing', () => {
|
|
64
|
+
const { stdout, status, stderr } = celilo('module list');
|
|
65
|
+
expect(status, `module list failed:\n${stderr}`).toBe(0);
|
|
66
|
+
|
|
67
|
+
// The exact shape celilo#695 tried and failed to match. No ANSI stripping,
|
|
68
|
+
// no prefix trimming — if this needs either, the bug is back.
|
|
69
|
+
const line = stdout.split('\n').find((l) => l.startsWith('celilo-mgmt '));
|
|
70
|
+
|
|
71
|
+
expect(
|
|
72
|
+
line,
|
|
73
|
+
`No line began with "celilo-mgmt ". stdout was:\n${JSON.stringify(stdout)}`,
|
|
74
|
+
).toBeDefined();
|
|
75
|
+
});
|
|
76
|
+
|
|
77
|
+
test('module list stdout carries no box-drawing or ANSI', () => {
|
|
78
|
+
const { stdout } = celilo('module list');
|
|
79
|
+
expect(BOX_DRAWING.test(stdout), `box-drawing in stdout:\n${JSON.stringify(stdout)}`).toBe(
|
|
80
|
+
false,
|
|
81
|
+
);
|
|
82
|
+
expect(ANSI.test(stdout), `ANSI in stdout:\n${JSON.stringify(stdout)}`).toBe(false);
|
|
83
|
+
});
|
|
84
|
+
|
|
85
|
+
test('a failing command writes its diagnostic to stderr, not stdout', () => {
|
|
86
|
+
const { stdout, stderr, status } = celilo('module where no-such-module-exists');
|
|
87
|
+
|
|
88
|
+
expect(status).not.toBe(0);
|
|
89
|
+
expect(stderr).toContain('Error');
|
|
90
|
+
// The whole point: stdout stays empty so a caller parsing it is not handed
|
|
91
|
+
// an error message where a result belongs.
|
|
92
|
+
expect(stdout.trim(), `error text leaked to stdout: ${JSON.stringify(stdout)}`).toBe('');
|
|
93
|
+
});
|
|
94
|
+
});
|
|
@@ -88,9 +88,11 @@ export const ALL_CATEGORIES: readonly DriftCategory[] = [
|
|
|
88
88
|
'disk_space',
|
|
89
89
|
'transport_reads',
|
|
90
90
|
'trusted_sources',
|
|
91
|
+
'interface_classification',
|
|
91
92
|
];
|
|
92
93
|
|
|
93
94
|
export const CATEGORY_LABELS: Record<DriftCategory, string> = {
|
|
95
|
+
interface_classification: 'Firewall interfaces',
|
|
94
96
|
cli_version: 'CLI version',
|
|
95
97
|
schema: 'Schema migrations',
|
|
96
98
|
capability_abi: 'Capability ABI',
|
package/src/cli/types.ts
CHANGED
|
@@ -20,8 +20,13 @@ export interface CommandSuccess {
|
|
|
20
20
|
message: string;
|
|
21
21
|
data?: unknown;
|
|
22
22
|
/**
|
|
23
|
-
*
|
|
24
|
-
*
|
|
23
|
+
* Marks a command whose `message` is a machine-readable payload (JSON, a
|
|
24
|
+
* unit file, a secret) rather than prose for a human.
|
|
25
|
+
*
|
|
26
|
+
* Every successful message now reaches stdout verbatim, so this no longer
|
|
27
|
+
* selects a different destination. It is kept as the contract: these are the
|
|
28
|
+
* commands a script parses, and nothing may ever decorate them (celilo#698,
|
|
29
|
+
* where ten JSON payloads were rendered unparseable by the formatter).
|
|
25
30
|
*/
|
|
26
31
|
rawOutput?: boolean;
|
|
27
32
|
}
|
package/src/db/schema.ts
CHANGED
|
@@ -11,7 +11,17 @@ import {
|
|
|
11
11
|
|
|
12
12
|
/**
|
|
13
13
|
* Module lifecycle states
|
|
14
|
-
* IMPORTED, VALIDATED, CONFIGURED, GENERATING, ERROR, DEPLOYING, INSTALLED, VERIFIED, UNINSTALLING
|
|
14
|
+
* IMPORTED, VALIDATED, CONFIGURED, GENERATING, ERROR, DEPLOYING, INSTALLED, VERIFIED, UNINSTALLING, PAUSED
|
|
15
|
+
*
|
|
16
|
+
* `PAUSED` is a real member of this union rather than a side flag, and that is
|
|
17
|
+
* the point (openspec/changes/module-pause-lifecycle/design.md D1): adding it
|
|
18
|
+
* makes the type-checker enumerate every site that must now consider
|
|
19
|
+
* paused-ness. A `pausedAt`-only flag would leave every `state === 'VERIFIED'`
|
|
20
|
+
* comparison silently compiling while quietly reading a paused module as live.
|
|
21
|
+
*
|
|
22
|
+
* There is deliberately no `prePauseState`: pause is legal only from a settled
|
|
23
|
+
* state and unpause redeploys, so the deploy path decides the resulting state
|
|
24
|
+
* and there is nothing to restore.
|
|
15
25
|
*/
|
|
16
26
|
export type ModuleState =
|
|
17
27
|
| 'IMPORTED'
|
|
@@ -22,23 +32,65 @@ export type ModuleState =
|
|
|
22
32
|
| 'INSTALLED'
|
|
23
33
|
| 'VERIFIED'
|
|
24
34
|
| 'ERROR'
|
|
25
|
-
| 'UNINSTALLING'
|
|
35
|
+
| 'UNINSTALLING'
|
|
36
|
+
| 'PAUSED';
|
|
37
|
+
|
|
38
|
+
/**
|
|
39
|
+
* States a module may be paused FROM (design D1). `ERROR` is deliberately
|
|
40
|
+
* included: quiescing a broken module to stop alert noise while working on it
|
|
41
|
+
* is legitimate, and refusing would push the operator toward silencing those
|
|
42
|
+
* alerts by some less visible route.
|
|
43
|
+
*/
|
|
44
|
+
export const PAUSABLE_STATES = [
|
|
45
|
+
'INSTALLED',
|
|
46
|
+
'VERIFIED',
|
|
47
|
+
'ERROR',
|
|
48
|
+
] as const satisfies readonly ModuleState[];
|
|
49
|
+
|
|
50
|
+
/**
|
|
51
|
+
* States that mean "a lifecycle transition is under way". Pausing one of these
|
|
52
|
+
* would strand the transition, so pause is refused with a distinct message from
|
|
53
|
+
* the never-deployed case.
|
|
54
|
+
*/
|
|
55
|
+
export const IN_FLIGHT_STATES = [
|
|
56
|
+
'GENERATING',
|
|
57
|
+
'DEPLOYING',
|
|
58
|
+
'UNINSTALLING',
|
|
59
|
+
] as const satisfies readonly ModuleState[];
|
|
26
60
|
|
|
27
61
|
/**
|
|
28
62
|
* Modules table - stores module metadata and manifest data
|
|
29
63
|
*/
|
|
30
|
-
export const modules = sqliteTable(
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
64
|
+
export const modules = sqliteTable(
|
|
65
|
+
'modules',
|
|
66
|
+
{
|
|
67
|
+
id: text('id').primaryKey(),
|
|
68
|
+
name: text('name').notNull(),
|
|
69
|
+
version: text('version').notNull(),
|
|
70
|
+
description: text('description'),
|
|
71
|
+
state: text('state').$type<ModuleState>().notNull().default('IMPORTED'),
|
|
72
|
+
manifestData: text('manifest_data', { mode: 'json' })
|
|
73
|
+
.$type<Record<string, unknown>>()
|
|
74
|
+
.notNull(),
|
|
75
|
+
sourcePath: text('source_path').notNull(),
|
|
76
|
+
importedAt: integer('imported_at', { mode: 'timestamp' }).notNull().default(sql`(unixepoch())`),
|
|
77
|
+
updatedAt: integer('updated_at', { mode: 'timestamp' }).notNull().default(sql`(unixepoch())`),
|
|
78
|
+
errorMessage: text('error_message'),
|
|
79
|
+
/**
|
|
80
|
+
* When the module was paused. Null unless `state = 'PAUSED'`. The state alone
|
|
81
|
+
* cannot answer "how long", and the DURATION is what makes a forgotten pause
|
|
82
|
+
* detectable (design D7) — every paused-module report carries the age.
|
|
83
|
+
*/
|
|
84
|
+
pausedAt: integer('paused_at', { mode: 'timestamp' }),
|
|
85
|
+
/** Operator-supplied explanation, so the row explains itself. */
|
|
86
|
+
pauseReason: text('pause_reason'),
|
|
87
|
+
},
|
|
88
|
+
(table) => ({
|
|
89
|
+
// Every management-API response asks "is anything paused?" (design D7), so
|
|
90
|
+
// that lookup must stay a single indexed hit rather than a table scan.
|
|
91
|
+
stateIdx: index('modules_state_idx').on(table.state),
|
|
92
|
+
}),
|
|
93
|
+
);
|
|
42
94
|
|
|
43
95
|
/**
|
|
44
96
|
* Module configuration - user-provided key-value pairs.
|
|
@@ -807,7 +859,13 @@ export const backups = sqliteTable('backups', {
|
|
|
807
859
|
* pid is no longer alive is treated as abandoned (the process crashed before
|
|
808
860
|
* the completion update landed) and ignored by in-flight checks.
|
|
809
861
|
*/
|
|
810
|
-
export type ModuleOperationKind =
|
|
862
|
+
export type ModuleOperationKind =
|
|
863
|
+
| 'deploy'
|
|
864
|
+
| 'uninstall'
|
|
865
|
+
| 'backup'
|
|
866
|
+
| 'restore'
|
|
867
|
+
| 'pause'
|
|
868
|
+
| 'unpause';
|
|
811
869
|
export type ModuleOperationStatus = 'in_progress' | 'completed' | 'failed';
|
|
812
870
|
|
|
813
871
|
export const moduleOperations = sqliteTable('module_operations', {
|
|
@@ -154,6 +154,32 @@ export async function resolveFirewallNatIp(db: DbClient): Promise<string | undef
|
|
|
154
154
|
return undefined;
|
|
155
155
|
}
|
|
156
156
|
|
|
157
|
+
/**
|
|
158
|
+
* The addresses of every machine a `firewall` capability provider manages.
|
|
159
|
+
*
|
|
160
|
+
* This is the authoritative answer to "which machines are firewalls", and it has
|
|
161
|
+
* to be: a machine's stored `role` is decided by `machine add`, from the zones
|
|
162
|
+
* declared AT THAT MOMENT. The normal order is `machine add` and THEN deploy
|
|
163
|
+
* iptables, whose `on_install` writes the zone subnets — so the firewall is
|
|
164
|
+
* recorded as a plain host and stays that way. A firewall provider naming an IP
|
|
165
|
+
* is a statement, not an inference from a snapshot.
|
|
166
|
+
*/
|
|
167
|
+
export async function listFirewallIps(db: DbClient): Promise<string[]> {
|
|
168
|
+
const providers = db
|
|
169
|
+
.select()
|
|
170
|
+
.from(capabilities)
|
|
171
|
+
.where(eq(capabilities.capabilityName, 'firewall'))
|
|
172
|
+
.all();
|
|
173
|
+
const ips: string[] = [];
|
|
174
|
+
for (const provider of providers) {
|
|
175
|
+
const config = await loadModuleConfig(provider.moduleId, db);
|
|
176
|
+
if (typeof config.firewall_ip === 'string' && config.firewall_ip) {
|
|
177
|
+
ips.push(config.firewall_ip);
|
|
178
|
+
}
|
|
179
|
+
}
|
|
180
|
+
return ips;
|
|
181
|
+
}
|
|
182
|
+
|
|
157
183
|
/**
|
|
158
184
|
* Caddy's zone-routable IP — its own DMZ ingress address (`target_ip`, the same
|
|
159
185
|
* value public_web exposes as `dmz_ip`). This is the in-zone split-horizon
|
|
@@ -606,6 +632,9 @@ function buildCapabilityInterface(
|
|
|
606
632
|
undefined, // no upstream — the chain path handles that
|
|
607
633
|
undefined, // logger is applied by wrapWithLogging at the loader site
|
|
608
634
|
trustedSourceStore,
|
|
635
|
+
// LIVE, for the same reason as the chain path: a declaration written
|
|
636
|
+
// during this hook run must be visible to the converge that follows it.
|
|
637
|
+
zones ? { list: zones.declaredNetworks } : undefined,
|
|
609
638
|
);
|
|
610
639
|
return stampProvider(iface, providerModuleId);
|
|
611
640
|
}
|
|
@@ -707,6 +736,18 @@ interface FirewallZones {
|
|
|
707
736
|
trustedSubnets: string[];
|
|
708
737
|
/** celilo's control-plane network, as a DESTINATION for trusted sources. */
|
|
709
738
|
controlPlaneSubnet?: string;
|
|
739
|
+
/**
|
|
740
|
+
* Every zone with a declared subnet, WITH its name — the input to interface
|
|
741
|
+
* classification. Excludes `external`, which is a residual rather than a
|
|
742
|
+
* subnet (design D1, amended).
|
|
743
|
+
*/
|
|
744
|
+
/**
|
|
745
|
+
* A LIVE read, not a snapshot. Everything else here is captured when the
|
|
746
|
+
* capability is built — before any hook runs — and for declarations that is
|
|
747
|
+
* wrong: `wireguard` declares its VPN subnet and brings `wg0` up inside one
|
|
748
|
+
* hook run, and the converge that follows has to see the declaration.
|
|
749
|
+
*/
|
|
750
|
+
declaredNetworks: () => Array<{ zone: string; subnet: string }>;
|
|
710
751
|
/**
|
|
711
752
|
* Every declared zone subnet — read from
|
|
712
753
|
* `network.<zone>.subnet` across all of `NETWORK_ZONES`, not just the three
|
|
@@ -806,6 +847,35 @@ export function loadTrustedSubnets(db: DbClient, firewallIp?: string): TrustedSu
|
|
|
806
847
|
* of them. Zones with no configured subnet are omitted — their traffic stays
|
|
807
848
|
* denied (fail-closed).
|
|
808
849
|
*/
|
|
850
|
+
/**
|
|
851
|
+
* Every network celilo can attribute an interface to: one entry per
|
|
852
|
+
* `network.<name>.subnet` in system config.
|
|
853
|
+
*
|
|
854
|
+
* Read from the config rather than from `NETWORK_ZONES`, because celilo declares
|
|
855
|
+
* networks that are not placement zones. `wireguard`'s `on_install` writes
|
|
856
|
+
* `network.control-plane-vpn.subnet` before it brings `wg0` up, exactly as the
|
|
857
|
+
* design requires (D3), and the firewall is supposed to classify `wg0` "by the
|
|
858
|
+
* same subnet containment it uses for every other leg". Reading only
|
|
859
|
+
* `NETWORK_ZONES` left that declaration invisible: the converge could not
|
|
860
|
+
* attribute `wg0` and refused — and on a firewall with a baseline it would have
|
|
861
|
+
* ISOLATED it, shutting down the admin VPN.
|
|
862
|
+
*
|
|
863
|
+
* `external` is deliberately absent even when something has set
|
|
864
|
+
* `network.external.subnet`: it is the RESIDUAL, decided by `isPubliclyRoutable`
|
|
865
|
+
* and never by containment (design D1, amended). Matching an interface to
|
|
866
|
+
* `external` by subnet would reintroduce the overload this change removes.
|
|
867
|
+
*/
|
|
868
|
+
export function readDeclaredNetworks(db: DbClient): Array<{ zone: string; subnet: string }> {
|
|
869
|
+
return db
|
|
870
|
+
.select()
|
|
871
|
+
.from(systemConfig)
|
|
872
|
+
.all()
|
|
873
|
+
.flatMap((row) => {
|
|
874
|
+
const zone = /^network\.(.+)\.subnet$/.exec(row.key)?.[1];
|
|
875
|
+
return zone && zone !== 'external' && row.value ? [{ zone, subnet: row.value }] : [];
|
|
876
|
+
});
|
|
877
|
+
}
|
|
878
|
+
|
|
809
879
|
function loadFirewallZones(db: DbClient): FirewallZones {
|
|
810
880
|
const zoneTiers: Array<{ name: string; subnet: string }> = [];
|
|
811
881
|
for (const zone of ZONE_TIER_ORDER) {
|
|
@@ -822,17 +892,65 @@ function loadFirewallZones(db: DbClient): FirewallZones {
|
|
|
822
892
|
// snapshot put the subnet straight back, so the set could grow but never
|
|
823
893
|
// shrink. That is exactly what happened: `on_uninstall` withdrew the VPN's
|
|
824
894
|
// trusted source, logged success, and the reach rules were re-rendered anyway.
|
|
895
|
+
// Every zone that HAS a declared subnet, keeping the zone NAME. Interface
|
|
896
|
+
// classification needs the name — `external` and `internal` are the only two
|
|
897
|
+
// legs a default route may leave through (design D11), which is not a question
|
|
898
|
+
// a bare list of subnets can answer.
|
|
899
|
+
//
|
|
900
|
+
// `external` is deliberately absent from this list even when something has set
|
|
901
|
+
// `network.external.subnet`: `external` is the RESIDUAL, decided by
|
|
902
|
+
// `isPubliclyRoutable`, never by containment (design D1, amended). Matching an
|
|
903
|
+
// interface to `external` by subnet would reintroduce the overload this change
|
|
904
|
+
// removes.
|
|
905
|
+
//
|
|
906
|
+
// EVERY declared network, not just the six `NETWORK_ZONES`. celilo declares
|
|
907
|
+
// networks that are not placement zones — `network.control-plane-vpn.subnet`,
|
|
908
|
+
// written by `wireguard`'s `on_install` — and the design is explicit that the
|
|
909
|
+
// firewall "classifies `wg0` by the same subnet containment it uses for every
|
|
910
|
+
// other leg". Reading only `NETWORK_ZONES` left that declaration invisible:
|
|
911
|
+
// the module declared the subnet before bringing the interface up, exactly as
|
|
912
|
+
// designed, and the converge still could not attribute `wg0` and refused. On a
|
|
913
|
+
// firewall that already had a baseline it would have gone further and ISOLATED
|
|
914
|
+
// the interface — celilo shutting down the admin VPN, which for a remote
|
|
915
|
+
// operator is the way back in.
|
|
916
|
+
//
|
|
917
|
+
// So the source is the config itself. A hardcoded list here could only ever
|
|
918
|
+
// describe the networks celilo knew about when this line was written.
|
|
919
|
+
|
|
825
920
|
return {
|
|
826
921
|
zoneTiers,
|
|
827
922
|
trustedSubnets: loadTrustedSubnets(db).map((e) => e.subnet),
|
|
828
923
|
controlPlaneSubnet: loadControlPlaneSubnet(db) ?? readZoneSubnet(db, 'internal'),
|
|
829
|
-
|
|
830
|
-
|
|
831
|
-
|
|
832
|
-
|
|
924
|
+
declaredNetworks: () => readDeclaredNetworks(db),
|
|
925
|
+
// Placement ZONES only — deliberately NOT `declaredNetworks`, which is wider.
|
|
926
|
+
// Fronting a subnet renders translation for it; the control-plane VPN
|
|
927
|
+
// reaches the fleet as a trusted source instead, and giving it a second
|
|
928
|
+
// mechanism would change the rendered ruleset for every existing fleet.
|
|
929
|
+
frontedSubnets: NETWORK_ZONES.filter((zone) => zone !== 'external').flatMap((zone) => {
|
|
930
|
+
const subnet = readZoneSubnet(db, zone);
|
|
931
|
+
return subnet ? [subnet] : [];
|
|
932
|
+
}),
|
|
833
933
|
};
|
|
834
934
|
}
|
|
835
935
|
|
|
936
|
+
/**
|
|
937
|
+
* The interface baseline as stored in module config: a comma-separated list of
|
|
938
|
+
* interface names, or absent.
|
|
939
|
+
*
|
|
940
|
+
* Absent and empty are the SAME thing here and both mean "no baseline" — a
|
|
941
|
+
* firewall with zero interfaces does not exist, so an empty string can only be
|
|
942
|
+
* a cleared or never-written value. Treating it as a baseline of nothing would
|
|
943
|
+
* mean every interface is "new", which is the opposite of what it says.
|
|
944
|
+
*/
|
|
945
|
+
function parseInterfaceBaseline(raw: unknown): string[] | undefined {
|
|
946
|
+
if (typeof raw !== 'string') return undefined;
|
|
947
|
+
const names = raw
|
|
948
|
+
.split(',')
|
|
949
|
+
.map((n) => n.trim())
|
|
950
|
+
.filter((n) => n.length > 0);
|
|
951
|
+
return names.length > 0 ? names : undefined;
|
|
952
|
+
}
|
|
953
|
+
|
|
836
954
|
async function buildFirewallChain(
|
|
837
955
|
allProviders: Array<{
|
|
838
956
|
id: number;
|
|
@@ -982,11 +1100,19 @@ async function buildFirewallChain(
|
|
|
982
1100
|
trustedSubnets: zones.trustedSubnets,
|
|
983
1101
|
controlPlaneSubnet: zones.controlPlaneSubnet,
|
|
984
1102
|
frontedSubnets: zones.frontedSubnets,
|
|
1103
|
+
// The recorded baseline (D12), from this firewall's own module config.
|
|
1104
|
+
// Absent means the box has never converged cleanly, so an interface
|
|
1105
|
+
// celilo cannot attribute refuses rather than being disabled.
|
|
1106
|
+
interfaceBaseline: parseInterfaceBaseline(provConfig.interface_baseline),
|
|
985
1107
|
},
|
|
986
1108
|
store,
|
|
987
1109
|
currentUpstream,
|
|
988
1110
|
logger,
|
|
989
1111
|
trustedSourceStore,
|
|
1112
|
+
// LIVE, not a snapshot: a declaration written during this hook run —
|
|
1113
|
+
// `wireguard` declaring its VPN subnet before creating `wg0` — must be
|
|
1114
|
+
// visible to the converge that follows it in the same run.
|
|
1115
|
+
{ list: zones.declaredNetworks },
|
|
990
1116
|
);
|
|
991
1117
|
|
|
992
1118
|
debugLog(`firewall chain: wired ${provider.moduleId} → ${hasExternal.moduleId}`);
|
|
@@ -49,6 +49,13 @@ export interface RunNamedHookResult extends HookResult {
|
|
|
49
49
|
* gracefully when a module has no `on_uninstall` defined.
|
|
50
50
|
*/
|
|
51
51
|
notDefined?: boolean;
|
|
52
|
+
/**
|
|
53
|
+
* True when the hook was not run because the module is PAUSED. Reported as
|
|
54
|
+
* success rather than failure: the module is deliberately quiesced, and a
|
|
55
|
+
* failure here would be retried by the bus and then alerted on — paging the
|
|
56
|
+
* operator about the pause they took themselves.
|
|
57
|
+
*/
|
|
58
|
+
skippedPaused?: boolean;
|
|
52
59
|
}
|
|
53
60
|
|
|
54
61
|
/**
|
|
@@ -82,6 +89,27 @@ export async function runNamedHook(
|
|
|
82
89
|
};
|
|
83
90
|
}
|
|
84
91
|
|
|
92
|
+
// Quiescence for a paused module (openspec/changes/module-pause-lifecycle,
|
|
93
|
+
// tasks 2.1/2.2). This is the chokepoint every non-lifecycle invocation
|
|
94
|
+
// funnels through — bus dispatch, timer fan-out, aspect fan-out,
|
|
95
|
+
// public-web republish, the dns-provider backfill, and `module run-hook` —
|
|
96
|
+
// so guarding here covers the paths individually rather than each caller
|
|
97
|
+
// remembering to.
|
|
98
|
+
//
|
|
99
|
+
// The exemptions are decided from the hook NAME, not a caller-supplied flag
|
|
100
|
+
// (Rule 10.3): `on_install` is how unpause redeploys the module back to life,
|
|
101
|
+
// and `on_uninstall` is how a paused module is removed — which is the entire
|
|
102
|
+
// point of pausing it. Both must run while `state` is still PAUSED.
|
|
103
|
+
const LIFECYCLE_HOOKS: readonly HookName[] = ['on_install', 'on_uninstall'];
|
|
104
|
+
if (module.state === 'PAUSED' && !LIFECYCLE_HOOKS.includes(hookName)) {
|
|
105
|
+
return {
|
|
106
|
+
success: true,
|
|
107
|
+
outputs: {},
|
|
108
|
+
duration: Date.now() - startedAt,
|
|
109
|
+
skippedPaused: true,
|
|
110
|
+
};
|
|
111
|
+
}
|
|
112
|
+
|
|
85
113
|
const manifest = module.manifestData as ModuleManifest;
|
|
86
114
|
const hookDef = manifest.hooks?.[hookName as keyof typeof manifest.hooks];
|
|
87
115
|
if (!hookDef) {
|
package/src/hooks/types.ts
CHANGED
|
@@ -120,7 +120,8 @@ export type HookName =
|
|
|
120
120
|
| 'on_backup_analyze'
|
|
121
121
|
| 'on_restore'
|
|
122
122
|
| 'on_system_event'
|
|
123
|
-
| 'refresh_registrations'
|
|
123
|
+
| 'refresh_registrations'
|
|
124
|
+
| 'reassert_dhcp_dns';
|
|
124
125
|
|
|
125
126
|
/**
|
|
126
127
|
* Hook manifest section - maps hook names to definitions
|
|
@@ -194,6 +194,22 @@ export const V1_HOOKS: ContractHooks = {
|
|
|
194
194
|
},
|
|
195
195
|
outputs: {},
|
|
196
196
|
},
|
|
197
|
+
/**
|
|
198
|
+
* Periodic re-assertion of the resolver a dns_internal provider hands
|
|
199
|
+
* out over DHCP (celilo#739). Some routers regenerate that value from
|
|
200
|
+
* their own upstream list on a timer and silently undo what on_install
|
|
201
|
+
* set — the write succeeds and reads back correct, so only a client
|
|
202
|
+
* renewing later ever sees the wrong resolver. No framework-injected
|
|
203
|
+
* inputs: the hook derives the address it wants from its own config and
|
|
204
|
+
* systems, and reads the device before writing. Typically driven by a
|
|
205
|
+
* `timer.tick.1m` subscription, because the tick interval is the
|
|
206
|
+
* worst-case window in which a renewing client can be handed the wrong
|
|
207
|
+
* resolver for a full lease.
|
|
208
|
+
*/
|
|
209
|
+
reassert_dhcp_dns: {
|
|
210
|
+
inputs: {},
|
|
211
|
+
outputs: {},
|
|
212
|
+
},
|
|
197
213
|
/**
|
|
198
214
|
* Build-bus upstream publish hook. The executor passes the
|
|
199
215
|
* PublishEvent fields as env vars (CELILO_EVENT_PAYLOAD,
|
package/src/manifest/schema.ts
CHANGED
|
@@ -1,5 +1,10 @@
|
|
|
1
1
|
import { z } from 'zod';
|
|
2
2
|
import { NETWORK_ZONES } from '../db/schema';
|
|
3
|
+
import {
|
|
4
|
+
BACKUP_CADENCE_FLOOR_MINUTES,
|
|
5
|
+
MONITOR_INTERVAL_FLOOR_MINUTES,
|
|
6
|
+
cadenceSchema,
|
|
7
|
+
} from '../services/cadence';
|
|
3
8
|
import { SUPPORTED_CONTRACT_VERSIONS } from './contracts';
|
|
4
9
|
|
|
5
10
|
/**
|
|
@@ -273,73 +278,26 @@ export const LifecycleHookSchema = z.object({
|
|
|
273
278
|
timeout: z.number().positive().optional(),
|
|
274
279
|
});
|
|
275
280
|
|
|
276
|
-
/**
|
|
277
|
-
* Floor for a module's suggested `health_check` interval.
|
|
278
|
-
*
|
|
279
|
-
* The monitor sweep rides the event bus's fixed `timer.tick.5m`
|
|
280
|
-
* (packages/event-bus/src/timer.ts), so an interval below five minutes
|
|
281
|
-
* cannot be honoured — it would silently round up. Rejecting it at manifest
|
|
282
|
-
* validation is better than accepting a promise celilo can't keep.
|
|
283
|
-
*/
|
|
284
|
-
export const MONITOR_INTERVAL_FLOOR_MINUTES = 5;
|
|
285
|
-
|
|
286
|
-
const DURATION_PATTERN = /^(\d+)(m|h|d)$/;
|
|
287
|
-
|
|
288
|
-
/**
|
|
289
|
-
* Parse a duration string (`15m`, `1h`, `1d`) to whole minutes.
|
|
290
|
-
* Returns null when the string is not a well-formed duration.
|
|
291
|
-
*/
|
|
292
|
-
export function parseIntervalMinutes(value: string): number | null {
|
|
293
|
-
const match = DURATION_PATTERN.exec(value);
|
|
294
|
-
if (!match) return null;
|
|
295
|
-
const amount = Number.parseInt(match[1], 10);
|
|
296
|
-
if (!Number.isFinite(amount) || amount <= 0) return null;
|
|
297
|
-
const unit = match[2];
|
|
298
|
-
if (unit === 'm') return amount;
|
|
299
|
-
if (unit === 'h') return amount * 60;
|
|
300
|
-
return amount * 60 * 24;
|
|
301
|
-
}
|
|
302
|
-
|
|
303
281
|
/**
|
|
304
282
|
* `health_check` accepts everything a lifecycle hook does, plus an optional
|
|
305
283
|
* `interval` — the module author's SUGGESTED monitoring cadence.
|
|
306
284
|
*
|
|
307
|
-
* It is only a suggestion: the operator's `
|
|
308
|
-
*
|
|
309
|
-
*
|
|
310
|
-
*
|
|
285
|
+
* It is only a suggestion: the operator's `health_check_interval` override
|
|
286
|
+
* decides, and resolution happens at read time so a corrected suggestion
|
|
287
|
+
* reaches every install that has not overridden
|
|
288
|
+
* ([[services/alerting/health-cadence.ts]]). A module that omits it is not
|
|
289
|
+
* monitored until someone names a cadence.
|
|
290
|
+
*
|
|
291
|
+
* The same spellings as every other cadence in celilo — a named period, a
|
|
292
|
+
* duration, or `manual` — with the floor derived from the alerting sweep's
|
|
293
|
+
* tick.
|
|
311
294
|
*/
|
|
312
295
|
export const HealthCheckHookSchema = LifecycleHookSchema.extend({
|
|
313
|
-
|
|
314
|
-
|
|
315
|
-
|
|
316
|
-
|
|
317
|
-
|
|
318
|
-
// sharing DURATION_PATTERN.
|
|
319
|
-
interval: z
|
|
320
|
-
.string()
|
|
321
|
-
.regex(DURATION_PATTERN)
|
|
322
|
-
.optional()
|
|
323
|
-
.describe(
|
|
324
|
-
'Suggested monitoring cadence, e.g. "15m", "1h", "1d". Must be 5m or longer — the monitor sweep runs on a 5-minute grid.',
|
|
325
|
-
)
|
|
326
|
-
.superRefine((value, ctx) => {
|
|
327
|
-
if (value === undefined) return;
|
|
328
|
-
const minutes = parseIntervalMinutes(value);
|
|
329
|
-
if (minutes === null) {
|
|
330
|
-
ctx.addIssue({
|
|
331
|
-
code: z.ZodIssueCode.custom,
|
|
332
|
-
message: `Invalid health_check interval "${value}". Use a duration like "15m", "1h", or "1d".`,
|
|
333
|
-
});
|
|
334
|
-
return;
|
|
335
|
-
}
|
|
336
|
-
if (minutes < MONITOR_INTERVAL_FLOOR_MINUTES) {
|
|
337
|
-
ctx.addIssue({
|
|
338
|
-
code: z.ZodIssueCode.custom,
|
|
339
|
-
message: `health_check interval "${value}" is below the ${MONITOR_INTERVAL_FLOOR_MINUTES}m floor — the monitor sweep runs on a ${MONITOR_INTERVAL_FLOOR_MINUTES}-minute grid and cannot run it more often. Use "${MONITOR_INTERVAL_FLOOR_MINUTES}m" or longer.`,
|
|
340
|
-
});
|
|
341
|
-
}
|
|
342
|
-
}),
|
|
296
|
+
interval: cadenceSchema({
|
|
297
|
+
floorMinutes: MONITOR_INTERVAL_FLOOR_MINUTES,
|
|
298
|
+
description:
|
|
299
|
+
'Suggested monitoring cadence: a duration like "15m", "1h" or "1d", a named period (hourly, daily, weekly, monthly), or "manual". The monitor sweep runs on a 5-minute grid, so nothing finer than 5m can be served.',
|
|
300
|
+
}).optional(),
|
|
343
301
|
});
|
|
344
302
|
|
|
345
303
|
/**
|
|
@@ -639,6 +597,16 @@ export const ModuleManifestSchema = z
|
|
|
639
597
|
* designs/DISPATCHER_DAEMON_AND_TIMER_EVENTS.md (B3).
|
|
640
598
|
*/
|
|
641
599
|
refresh_registrations: LifecycleHookSchema.optional(),
|
|
600
|
+
/**
|
|
601
|
+
* Periodic re-assertion of the resolver a dns_internal provider
|
|
602
|
+
* hands out over DHCP. Some routers regenerate that value from
|
|
603
|
+
* their own upstream list on a timer, silently undoing what
|
|
604
|
+
* on_install set. The hook reads the device before writing, so a
|
|
605
|
+
* quiet minute costs one query. Subscribe it to `timer.tick.1m` —
|
|
606
|
+
* the tick interval IS the worst-case window in which a renewing
|
|
607
|
+
* client can be handed the wrong resolver. See celilo#739.
|
|
608
|
+
*/
|
|
609
|
+
reassert_dhcp_dns: LifecycleHookSchema.optional(),
|
|
642
610
|
/**
|
|
643
611
|
* Build-bus upstream publish hooks. Array (a module can react
|
|
644
612
|
* to multiple upstream packages with different actions). See
|
|
@@ -694,10 +662,17 @@ export const ModuleManifestSchema = z
|
|
|
694
662
|
|
|
695
663
|
backup: z
|
|
696
664
|
.object({
|
|
697
|
-
//
|
|
698
|
-
//
|
|
699
|
-
// [[services/backup-schedule.ts]].
|
|
700
|
-
|
|
665
|
+
// The author's SUGGESTED cadence. The operator's `backup_schedule`
|
|
666
|
+
// override wins and is resolved at read time — see
|
|
667
|
+
// [[services/backup-schedule.ts]]. Absent from both means `daily`;
|
|
668
|
+
// opting out of backups entirely is a real decision and takes an
|
|
669
|
+
// explicit `manual`.
|
|
670
|
+
//
|
|
671
|
+
schedule: cadenceSchema({
|
|
672
|
+
floorMinutes: BACKUP_CADENCE_FLOOR_MINUTES,
|
|
673
|
+
description:
|
|
674
|
+
'Suggested backup cadence: a named period (hourly, daily, weekly, monthly), a duration like "6h", or "manual" to opt out. The backup sweep runs hourly, so nothing finer than 1h can be served.',
|
|
675
|
+
}).default('daily'),
|
|
701
676
|
retention: z
|
|
702
677
|
.object({
|
|
703
678
|
count: z.number().int().positive().default(7),
|