@celilo/cli 0.23.0 → 0.24.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CELILO_CORE_MODULES.md +2 -2
- package/CELILO_SUBSYSTEMS.md +27 -7
- package/package.json +6 -5
- package/src/cli/commands/alerts-act.ts +1 -1
- package/src/cli/commands/backup-create.ts +26 -11
- package/src/cli/commands/backup-list.test.ts +83 -0
- package/src/cli/commands/backup-list.ts +67 -3
- package/src/cli/commands/backup-prune.ts +17 -17
- package/src/cli/commands/backup-sweep.ts +20 -8
- package/src/cli/commands/firewall-interface-list.test.ts +85 -0
- package/src/cli/commands/firewall-interface-list.ts +123 -0
- package/src/cli/commands/machine-add.ts +30 -2
- package/src/cli/commands/module-config.test.ts +64 -2
- package/src/cli/commands/module-config.ts +159 -8
- package/src/cli/commands/module-status.ts +124 -0
- package/src/cli/commands/monitor.ts +116 -19
- package/src/cli/commands/system-migrate.ts +14 -0
- package/src/cli/commands/system-update.ts +4 -1
- package/src/cli/completion.ts +35 -9
- package/src/cli/index.ts +59 -2
- package/src/cli/tui/audit-state.ts +2 -0
- package/src/hooks/capability-loader.ts +130 -4
- package/src/hooks/types.ts +2 -1
- package/src/manifest/contracts/v1.ts +16 -0
- package/src/manifest/schema.ts +40 -65
- package/src/services/alerting/builtin-monitors.test.ts +18 -10
- package/src/services/alerting/cadence-migration.test.ts +155 -0
- package/src/services/alerting/cadence-migration.ts +90 -0
- package/src/services/alerting/coverage-source.ts +8 -11
- package/src/services/alerting/deploy-hooks.test.ts +16 -7
- package/src/services/alerting/deploy-hooks.ts +11 -5
- package/src/services/alerting/health-cadence.test.ts +58 -0
- package/src/services/alerting/health-cadence.ts +128 -0
- package/src/services/alerting/health-coverage.ts +18 -8
- package/src/services/alerting/monitors.ts +50 -15
- package/src/services/alerting/sweep-runner.test.ts +51 -3
- package/src/services/alerting/sweep-runner.ts +30 -7
- package/src/services/audit/backup-source.ts +24 -1
- package/src/services/audit/backups.test.ts +95 -10
- package/src/services/audit/backups.ts +40 -37
- package/src/services/audit/interface-classification.test.ts +220 -0
- package/src/services/audit/interface-classification.ts +167 -0
- package/src/services/audit/types.ts +2 -1
- package/src/services/backup-age-agreement.test.ts +118 -0
- package/src/services/backup-create.ts +36 -30
- package/src/services/backup-metadata.ts +52 -1
- package/src/services/backup-retention.test.ts +123 -0
- package/src/services/backup-retention.ts +66 -5
- package/src/services/backup-schedule.test.ts +166 -0
- package/src/services/backup-schedule.ts +105 -15
- package/src/services/backup-staging.ts +14 -1
- package/src/services/backup-sweep.test.ts +22 -3
- package/src/services/backup-sweep.ts +15 -5
- package/src/services/cadence.test.ts +97 -0
- package/src/services/cadence.ts +165 -0
- package/src/services/machine-detector.ts +23 -1
- package/src/services/module-config.ts +33 -0
- package/src/services/storage-providers/s3.test.ts +96 -13
- package/src/services/storage-providers/s3.ts +48 -15
- package/src/services/zone-detector.test.ts +34 -3
- package/src/services/zone-detector.ts +33 -13
|
@@ -0,0 +1,123 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* `celilo firewall interface list [<hostname>]`
|
|
3
|
+
*
|
|
4
|
+
* The interface classification, on demand, with no side effects.
|
|
5
|
+
*
|
|
6
|
+
* A converge already refuses or isolates on what it finds — but only when it
|
|
7
|
+
* runs, and only in the middle of a deploy's output. An operator about to
|
|
8
|
+
* onboard a firewall, or wondering why one refused, needs to be able to ASK.
|
|
9
|
+
* The condition that produced `fw-keeper.sh` was not that celilo lacked the
|
|
10
|
+
* information; it was that celilo never said it.
|
|
11
|
+
*
|
|
12
|
+
* Read-only by construction: it reads the stored interface table and the
|
|
13
|
+
* declarations, and classifies in memory. It never touches the box, so it is
|
|
14
|
+
* safe to run against a firewall that is currently refusing.
|
|
15
|
+
*/
|
|
16
|
+
|
|
17
|
+
import { classifyInterfaces, isPubliclyRoutable } from '@celilo/capabilities';
|
|
18
|
+
import { getDb } from '../../db/client';
|
|
19
|
+
import { listFirewallIps, readDeclaredNetworks } from '../../hooks/capability-loader';
|
|
20
|
+
import { listMachines } from '../../services/machine-pool';
|
|
21
|
+
import { celiloIntro } from '../prompts';
|
|
22
|
+
import type { CommandResult } from '../types';
|
|
23
|
+
|
|
24
|
+
/**
|
|
25
|
+
* Every network a subnet is declared for — THE same reader the converge uses.
|
|
26
|
+
*
|
|
27
|
+
* This command's whole value is telling an operator what the next converge will
|
|
28
|
+
* do, so reading declarations a second way is not a duplication smell, it is a
|
|
29
|
+
* correctness bug: this file walked `NETWORK_ZONES` and so could not see
|
|
30
|
+
* `network.control-plane-vpn.subnet`. It would have reported `wg0` as ALIEN —
|
|
31
|
+
* "this will be isolated" — about an interface the converge attributes and
|
|
32
|
+
* leaves alone. The operator's most likely response to that reading is to go
|
|
33
|
+
* and remove their own admin VPN.
|
|
34
|
+
*/
|
|
35
|
+
async function declaredZones(): Promise<Array<{ zone: string; subnet: string }>> {
|
|
36
|
+
return readDeclaredNetworks(getDb());
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
/** One line per interface, explaining the role rather than just naming it. */
|
|
40
|
+
function describe(role: string, zone: string | undefined, ip: string): string {
|
|
41
|
+
if (role === 'zone') return `zone:${zone}`;
|
|
42
|
+
if (role === 'external') return 'external — the WAN edge';
|
|
43
|
+
return isPubliclyRoutable(ip)
|
|
44
|
+
? 'ALIEN — publicly routable but no declared zone claims it'
|
|
45
|
+
: 'ALIEN — no declared subnet contains it';
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
export async function handleFirewallInterfaceList(
|
|
49
|
+
args: string[],
|
|
50
|
+
_flags: Record<string, boolean | string> = {},
|
|
51
|
+
): Promise<CommandResult> {
|
|
52
|
+
celiloIntro('Firewall interfaces');
|
|
53
|
+
|
|
54
|
+
const wanted = args[0];
|
|
55
|
+
const machines = await listMachines();
|
|
56
|
+
// A firewall is a machine a firewall provider MANAGES. `role === 'router'` is
|
|
57
|
+
// kept as a second way in, but it cannot be the only one: the role is decided
|
|
58
|
+
// by `machine add` from the zones declared at that moment, and the normal
|
|
59
|
+
// order is to add the machine and THEN deploy iptables, whose `on_install`
|
|
60
|
+
// writes the zone subnets. So a working firewall is recorded as a plain host,
|
|
61
|
+
// and this command — whose entire purpose is to report on firewalls — answered
|
|
62
|
+
// "No firewalls in the machine pool" on a fleet that had one.
|
|
63
|
+
//
|
|
64
|
+
// Named explicitly, the hostname wins, so an operator can inspect any box.
|
|
65
|
+
const firewallIps = new Set(await listFirewallIps(getDb()));
|
|
66
|
+
const targets = wanted
|
|
67
|
+
? machines.filter((m) => m.hostname === wanted)
|
|
68
|
+
: machines.filter((m) => firewallIps.has(m.ipAddress) || m.role === 'router');
|
|
69
|
+
|
|
70
|
+
if (targets.length === 0) {
|
|
71
|
+
return {
|
|
72
|
+
success: false,
|
|
73
|
+
error: wanted
|
|
74
|
+
? `No machine named "${wanted}". Run \`celilo machine list\` to see the pool.`
|
|
75
|
+
: 'No firewalls in the machine pool. celilo looks for a machine managed by a firewall provider, or one it classified as a router.',
|
|
76
|
+
};
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
const zones = await declaredZones();
|
|
80
|
+
if (zones.length === 0) {
|
|
81
|
+
console.log(
|
|
82
|
+
'No zone subnets are declared, so every interface will read as unaccounted for.\n' +
|
|
83
|
+
'Declare them with: celilo system config set network.<zone>.subnet <cidr>\n',
|
|
84
|
+
);
|
|
85
|
+
}
|
|
86
|
+
|
|
87
|
+
let alienTotal = 0;
|
|
88
|
+
for (const machine of targets) {
|
|
89
|
+
console.log(`\n${machine.hostname} (${machine.ipAddress})`);
|
|
90
|
+
|
|
91
|
+
const interfaces = (machine.interfaces ?? []).map((i) => ({
|
|
92
|
+
name: i.name,
|
|
93
|
+
ip: i.ipAddress,
|
|
94
|
+
}));
|
|
95
|
+
if (interfaces.length === 0) {
|
|
96
|
+
console.log(' no interfaces recorded — re-run `celilo machine add` to detect them');
|
|
97
|
+
continue;
|
|
98
|
+
}
|
|
99
|
+
|
|
100
|
+
for (const c of classifyInterfaces(interfaces, zones)) {
|
|
101
|
+
if (c.role === 'alien') alienTotal += 1;
|
|
102
|
+
console.log(` ${c.name.padEnd(8)} ${c.ip.padEnd(16)} ${describe(c.role, c.zone, c.ip)}`);
|
|
103
|
+
}
|
|
104
|
+
}
|
|
105
|
+
|
|
106
|
+
console.log('');
|
|
107
|
+
if (alienTotal > 0) {
|
|
108
|
+
// Say what will HAPPEN, not merely what was found — the answer differs by
|
|
109
|
+
// whether celilo has a baseline for the box, and that is the thing an
|
|
110
|
+
// operator most needs to know before the next converge.
|
|
111
|
+
const consequence = [
|
|
112
|
+
'On a firewall celilo has not yet converged cleanly, the next converge will REFUSE and change nothing.',
|
|
113
|
+
'On one with a recorded baseline, an interface that appeared since will be isolated.',
|
|
114
|
+
'Resolve either by declaring a zone: celilo system config set network.<zone>.subnet <cidr>',
|
|
115
|
+
].join('\n');
|
|
116
|
+
console.log(`${alienTotal} interface(s) celilo cannot attribute.\n${consequence}`);
|
|
117
|
+
}
|
|
118
|
+
|
|
119
|
+
return {
|
|
120
|
+
success: true,
|
|
121
|
+
message: alienTotal === 0 ? 'every interface accounted for' : `${alienTotal} unaccounted`,
|
|
122
|
+
};
|
|
123
|
+
}
|
|
@@ -6,6 +6,7 @@
|
|
|
6
6
|
import { existsSync } from 'node:fs';
|
|
7
7
|
import { readFileSync } from 'node:fs';
|
|
8
8
|
import { join } from 'node:path';
|
|
9
|
+
import { isPubliclyRoutable } from '@celilo/capabilities';
|
|
9
10
|
import { getDb } from '../../db/client';
|
|
10
11
|
import type { NetworkZone } from '../../db/schema';
|
|
11
12
|
import { askText, withInterviewSession } from '../../services/bus-interview';
|
|
@@ -16,6 +17,7 @@ import {
|
|
|
16
17
|
detectNetworkInterfacesLocal,
|
|
17
18
|
testSshConnection,
|
|
18
19
|
} from '../../services/machine-detector';
|
|
20
|
+
import { describeInterfaceZone } from '../../services/machine-detector';
|
|
19
21
|
import { addMachine, getMachineByIp } from '../../services/machine-pool';
|
|
20
22
|
import { loadExistingConfiguration } from '../../services/system-init';
|
|
21
23
|
import { detectZoneFromIp } from '../../services/zone-detector';
|
|
@@ -242,8 +244,34 @@ export async function handleMachineAdd(
|
|
|
242
244
|
console.log(` Disk: ${detectedInfo.hardware.disk_gb} GB\n`);
|
|
243
245
|
|
|
244
246
|
// Zone: explicit override, else infer from IP.
|
|
247
|
+
//
|
|
248
|
+
// `detectZoneFromIp` answers containment only, and now says `'unknown'`
|
|
249
|
+
// rather than claiming `external` when nothing matches. Resolving that
|
|
250
|
+
// is a SECOND question — is this address one the internet can route to?
|
|
251
|
+
// — and the two were conflated before, which is how a private address in
|
|
252
|
+
// no declared subnet got labelled as facing the internet.
|
|
245
253
|
console.log('Detecting network zone...');
|
|
246
|
-
|
|
254
|
+
if (zoneOverride) {
|
|
255
|
+
zone = zoneOverride;
|
|
256
|
+
} else {
|
|
257
|
+
const detected = await detectZoneFromIp(ipAddress);
|
|
258
|
+
if (detected !== 'unknown') {
|
|
259
|
+
zone = detected;
|
|
260
|
+
} else if (isPubliclyRoutable(ipAddress)) {
|
|
261
|
+
// No declared subnet contains it and the internet can route to it:
|
|
262
|
+
// that is what `external` means — a cloud/VPS box.
|
|
263
|
+
zone = 'external';
|
|
264
|
+
} else {
|
|
265
|
+
// A private address in no declared subnet is UN-ZONEABLE, not
|
|
266
|
+
// external. Guessing here is the original defect; ask instead.
|
|
267
|
+
const fix =
|
|
268
|
+
'Fix: pass --zone <zone>, or declare the subnet with `celilo system config set network.<zone>.subnet <cidr>` and retry.';
|
|
269
|
+
return {
|
|
270
|
+
success: false,
|
|
271
|
+
error: `Cannot infer a zone for ${ipAddress}: it is not publicly routable and no declared network.<zone>.subnet contains it.\n${fix}`,
|
|
272
|
+
};
|
|
273
|
+
}
|
|
274
|
+
}
|
|
247
275
|
console.log(`✓ Zone: ${zone}\n`);
|
|
248
276
|
|
|
249
277
|
// Detect network interfaces and classify machine
|
|
@@ -254,7 +282,7 @@ export async function handleMachineAdd(
|
|
|
254
282
|
|
|
255
283
|
console.log(`✓ Role: ${role}`);
|
|
256
284
|
for (const iface of interfaces) {
|
|
257
|
-
console.log(` ${iface.name}: ${iface.ipAddress} (${iface
|
|
285
|
+
console.log(` ${iface.name}: ${iface.ipAddress} (${describeInterfaceZone(iface)})`);
|
|
258
286
|
}
|
|
259
287
|
console.log('');
|
|
260
288
|
}
|
|
@@ -14,7 +14,9 @@ import { modules } from '../../db/schema';
|
|
|
14
14
|
import { resolveDeployPosture } from '../../services/deploy-posture';
|
|
15
15
|
import {
|
|
16
16
|
FRAMEWORK_CONFIG_KEYS,
|
|
17
|
+
handleModuleConfigGet,
|
|
17
18
|
handleModuleConfigSet,
|
|
19
|
+
handleModuleConfigUnset,
|
|
18
20
|
validateFrameworkConfigValue,
|
|
19
21
|
} from './module-config';
|
|
20
22
|
import { pickUpgradePolicy } from './module-upgrade';
|
|
@@ -112,6 +114,30 @@ describe('handleModuleConfigSet — infra-key contract (ISS-0069)', () => {
|
|
|
112
114
|
expect(result.success).toBe(false);
|
|
113
115
|
});
|
|
114
116
|
|
|
117
|
+
test('a per-module policy key is settable on a module whose manifest never mentions it', async () => {
|
|
118
|
+
expect((await handleModuleConfigSet(['testmod', 'backup_schedule', '6h'])).success).toBe(true);
|
|
119
|
+
const read = await handleModuleConfigGet(['testmod', 'backup_schedule']);
|
|
120
|
+
expect(read.success).toBe(true);
|
|
121
|
+
if (read.success) expect(read.message).toContain('6h');
|
|
122
|
+
});
|
|
123
|
+
|
|
124
|
+
test('unset removes the override so the module follows its manifest again', async () => {
|
|
125
|
+
await handleModuleConfigSet(['testmod', 'backup_schedule', 'weekly']);
|
|
126
|
+
|
|
127
|
+
const unset = await handleModuleConfigUnset(['testmod', 'backup_schedule']);
|
|
128
|
+
expect(unset.success).toBe(true);
|
|
129
|
+
|
|
130
|
+
expect((await handleModuleConfigGet(['testmod', 'backup_schedule'])).success).toBe(false);
|
|
131
|
+
});
|
|
132
|
+
|
|
133
|
+
test('unsetting a key that was never set reports it and still succeeds', async () => {
|
|
134
|
+
// `unset` states a desired end state. Failing on an already-clean one makes
|
|
135
|
+
// it unusable from any script that cannot check first.
|
|
136
|
+
const result = await handleModuleConfigUnset(['testmod', 'backup_schedule']);
|
|
137
|
+
expect(result.success).toBe(true);
|
|
138
|
+
if (result.success) expect(result.message).toContain('No override set');
|
|
139
|
+
});
|
|
140
|
+
|
|
115
141
|
test('the valid-keys hint advertises the celilo-managed keys', async () => {
|
|
116
142
|
const result = await handleModuleConfigSet(['testmod', 'nope', 'x']);
|
|
117
143
|
expect(result.success).toBe(false);
|
|
@@ -127,8 +153,19 @@ describe('validateFrameworkConfigValue (pure)', () => {
|
|
|
127
153
|
expect(validateFrameworkConfigValue('app_port', 'anything')).toBeNull();
|
|
128
154
|
});
|
|
129
155
|
|
|
130
|
-
test('accepts
|
|
131
|
-
|
|
156
|
+
test('every framework key accepts at least its documented values', () => {
|
|
157
|
+
const documented: Record<string, string[]> = {
|
|
158
|
+
auto_upgrade: ['true', 'false'],
|
|
159
|
+
upgrade_policy: ['by-semver', 'always-safe', 'always-fast'],
|
|
160
|
+
backup_schedule: ['hourly', 'daily', 'weekly', 'monthly', 'manual', '6h'],
|
|
161
|
+
health_check_interval: ['15m', '1h', 'daily', 'manual'],
|
|
162
|
+
backup_retention_count: ['1', '3', '30'],
|
|
163
|
+
backup_retention_max_age_days: ['1', '30', '365'],
|
|
164
|
+
};
|
|
165
|
+
// Every key must be covered, so adding one without deciding what it accepts
|
|
166
|
+
// fails here rather than shipping unvalidated.
|
|
167
|
+
expect(Object.keys(documented).sort()).toEqual(Object.keys(FRAMEWORK_CONFIG_KEYS).sort());
|
|
168
|
+
for (const [key, values] of Object.entries(documented)) {
|
|
132
169
|
for (const v of values) expect(validateFrameworkConfigValue(key, v)).toBeNull();
|
|
133
170
|
}
|
|
134
171
|
});
|
|
@@ -136,6 +173,31 @@ describe('validateFrameworkConfigValue (pure)', () => {
|
|
|
136
173
|
test('rejects an unlisted value', () => {
|
|
137
174
|
expect(validateFrameworkConfigValue('upgrade_policy', 'always_safe')).toContain('Allowed');
|
|
138
175
|
});
|
|
176
|
+
|
|
177
|
+
test('rejects a cadence the sweep that would serve it cannot run', () => {
|
|
178
|
+
// The backup sweep rides an hourly tick; accepting `5m` would leave the
|
|
179
|
+
// operator believing they configured something that can never happen.
|
|
180
|
+
expect(validateFrameworkConfigValue('backup_schedule', '5m')).toContain('hourly');
|
|
181
|
+
// The alerting sweep ticks every five minutes, so the same value is fine there.
|
|
182
|
+
expect(validateFrameworkConfigValue('health_check_interval', '5m')).toBeNull();
|
|
183
|
+
expect(validateFrameworkConfigValue('health_check_interval', '1m')).toContain('Allowed');
|
|
184
|
+
});
|
|
185
|
+
|
|
186
|
+
test('a retention of zero or below is refused rather than read as "keep nothing"', () => {
|
|
187
|
+
// A `0` read as a bound would delete every backup the module has.
|
|
188
|
+
for (const bad of ['0', '-1', '2.5', 'lots']) {
|
|
189
|
+
expect(validateFrameworkConfigValue('backup_retention_count', bad)).toContain('1 or greater');
|
|
190
|
+
expect(validateFrameworkConfigValue('backup_retention_max_age_days', bad)).toContain(
|
|
191
|
+
'1 or greater',
|
|
192
|
+
);
|
|
193
|
+
}
|
|
194
|
+
});
|
|
195
|
+
|
|
196
|
+
test('a misspelled cadence is refused, and says it was not coerced', () => {
|
|
197
|
+
const error = validateFrameworkConfigValue('backup_schedule', 'dailyy');
|
|
198
|
+
expect(error).toContain('named period');
|
|
199
|
+
expect(error).toContain('Rejected rather than coerced');
|
|
200
|
+
});
|
|
139
201
|
});
|
|
140
202
|
|
|
141
203
|
// The point of the whole control: with always-safe set, a PATCH upgrade — which
|
|
@@ -3,9 +3,25 @@
|
|
|
3
3
|
*/
|
|
4
4
|
|
|
5
5
|
import { eq } from 'drizzle-orm';
|
|
6
|
+
import { z } from 'zod';
|
|
6
7
|
import { getDb } from '../../db/client';
|
|
7
8
|
import { modules } from '../../db/schema';
|
|
8
9
|
import {
|
|
10
|
+
HEALTH_CHECK_INTERVAL_CONFIG_KEY,
|
|
11
|
+
reconcileModuleWatchState,
|
|
12
|
+
} from '../../services/alerting/health-cadence';
|
|
13
|
+
import {
|
|
14
|
+
BACKUP_RETENTION_COUNT_CONFIG_KEY,
|
|
15
|
+
BACKUP_RETENTION_MAX_AGE_DAYS_CONFIG_KEY,
|
|
16
|
+
} from '../../services/backup-retention';
|
|
17
|
+
import { BACKUP_SCHEDULE_CONFIG_KEY } from '../../services/backup-schedule';
|
|
18
|
+
import {
|
|
19
|
+
BACKUP_CADENCE_FLOOR_MINUTES,
|
|
20
|
+
MONITOR_INTERVAL_FLOOR_MINUTES,
|
|
21
|
+
cadenceSchema,
|
|
22
|
+
} from '../../services/cadence';
|
|
23
|
+
import {
|
|
24
|
+
deleteModuleConfig,
|
|
9
25
|
formatConfigValue,
|
|
10
26
|
getAllModuleConfigValues,
|
|
11
27
|
getModuleConfigValue,
|
|
@@ -18,6 +34,12 @@ import type { CommandResult } from '../types';
|
|
|
18
34
|
* Operator keys that EVERY module accepts, whether or not its manifest declares
|
|
19
35
|
* them, with their permitted values.
|
|
20
36
|
*
|
|
37
|
+
* This is also where a module's per-module POLICY lives: how often to back it
|
|
38
|
+
* up, how often to health-check it. A manifest states those as the author's
|
|
39
|
+
* suggestion about a fleet they have never seen; the row an operator writes
|
|
40
|
+
* here wins, and every reader resolves the two at read time so a corrected
|
|
41
|
+
* manifest still reaches installs that have not overridden it.
|
|
42
|
+
*
|
|
21
43
|
* These describe how celilo TREATS a module (its CD policy), not how the module
|
|
22
44
|
* configures itself, so gating them on `variables.owns` had it backwards: it
|
|
23
45
|
* required each module author to opt into being manageable. The failure was
|
|
@@ -34,9 +56,69 @@ import type { CommandResult } from '../types';
|
|
|
34
56
|
* floor while nothing changed. A safety control that fails open on a typo is
|
|
35
57
|
* worse than no control.
|
|
36
58
|
*/
|
|
37
|
-
export
|
|
38
|
-
|
|
39
|
-
|
|
59
|
+
export interface FrameworkConfigKey {
|
|
60
|
+
/** Accepts the value the operator typed, or explains what it should be. */
|
|
61
|
+
schema: z.ZodTypeAny;
|
|
62
|
+
/**
|
|
63
|
+
* Why this key is refused rather than coerced. Carried per key rather than
|
|
64
|
+
* derived from the schema's error: the substance is what a WRONG value would
|
|
65
|
+
* silently do, and no validator knows that.
|
|
66
|
+
*/
|
|
67
|
+
why: string;
|
|
68
|
+
}
|
|
69
|
+
|
|
70
|
+
const REFUSED_NOT_COERCED_UPGRADE =
|
|
71
|
+
'Rejected rather than coerced: an unrecognized value silently falls back to the PERMISSIVE default (upgrade_policy → by-semver, which skips the pre-deploy backup on a patch), so a typo would look like it took effect.';
|
|
72
|
+
|
|
73
|
+
const REFUSED_NOT_COERCED_CADENCE =
|
|
74
|
+
'Rejected rather than coerced: an unrecognized cadence falls back to the manifest\'s suggestion, so a typo would leave the module on the cadence you meant to change — visibly "set", and doing nothing.';
|
|
75
|
+
|
|
76
|
+
const REFUSED_NOT_COERCED_RETENTION =
|
|
77
|
+
'Rejected rather than coerced: an unrecognized retention value falls back to the manifest, so a typo would leave the module keeping a different number of backups than you asked for — and the direction that goes wrong deletes data.';
|
|
78
|
+
|
|
79
|
+
/** An enum whose rejection message reads as an operator instruction, not a type error. */
|
|
80
|
+
function oneOf(values: readonly [string, ...string[]]): z.ZodTypeAny {
|
|
81
|
+
return z.enum(values, { errorMap: () => ({ message: `Allowed: ${values.join(', ')}` }) });
|
|
82
|
+
}
|
|
83
|
+
|
|
84
|
+
/**
|
|
85
|
+
* A whole number of things, at least one. Zero is refused rather than read as
|
|
86
|
+
* "keep nothing": a retention of 0 would delete every backup the module has.
|
|
87
|
+
*/
|
|
88
|
+
function positiveInteger(what: string): z.ZodTypeAny {
|
|
89
|
+
return z.string().superRefine((value, ctx) => {
|
|
90
|
+
const parsed = Number(value);
|
|
91
|
+
if (!Number.isInteger(parsed) || parsed < 1) {
|
|
92
|
+
ctx.addIssue({
|
|
93
|
+
code: z.ZodIssueCode.custom,
|
|
94
|
+
message: `Allowed: a whole number of ${what}, 1 or greater. Unset the key to keep everything.`,
|
|
95
|
+
});
|
|
96
|
+
}
|
|
97
|
+
});
|
|
98
|
+
}
|
|
99
|
+
|
|
100
|
+
export const FRAMEWORK_CONFIG_KEYS: Record<string, FrameworkConfigKey> = {
|
|
101
|
+
auto_upgrade: { schema: oneOf(['true', 'false']), why: REFUSED_NOT_COERCED_UPGRADE },
|
|
102
|
+
upgrade_policy: {
|
|
103
|
+
schema: oneOf(['by-semver', 'always-safe', 'always-fast']),
|
|
104
|
+
why: REFUSED_NOT_COERCED_UPGRADE,
|
|
105
|
+
},
|
|
106
|
+
[BACKUP_SCHEDULE_CONFIG_KEY]: {
|
|
107
|
+
schema: cadenceSchema({ floorMinutes: BACKUP_CADENCE_FLOOR_MINUTES }),
|
|
108
|
+
why: REFUSED_NOT_COERCED_CADENCE,
|
|
109
|
+
},
|
|
110
|
+
[HEALTH_CHECK_INTERVAL_CONFIG_KEY]: {
|
|
111
|
+
schema: cadenceSchema({ floorMinutes: MONITOR_INTERVAL_FLOOR_MINUTES }),
|
|
112
|
+
why: REFUSED_NOT_COERCED_CADENCE,
|
|
113
|
+
},
|
|
114
|
+
[BACKUP_RETENTION_COUNT_CONFIG_KEY]: {
|
|
115
|
+
schema: positiveInteger('copies to keep'),
|
|
116
|
+
why: REFUSED_NOT_COERCED_RETENTION,
|
|
117
|
+
},
|
|
118
|
+
[BACKUP_RETENTION_MAX_AGE_DAYS_CONFIG_KEY]: {
|
|
119
|
+
schema: positiveInteger('days to keep a backup'),
|
|
120
|
+
why: REFUSED_NOT_COERCED_RETENTION,
|
|
121
|
+
},
|
|
40
122
|
};
|
|
41
123
|
|
|
42
124
|
/**
|
|
@@ -44,10 +126,12 @@ export const FRAMEWORK_CONFIG_KEYS: Record<string, readonly string[]> = {
|
|
|
44
126
|
* or null when the key is not a framework key or the value is permitted.
|
|
45
127
|
*/
|
|
46
128
|
export function validateFrameworkConfigValue(key: string, value: string): string | null {
|
|
47
|
-
const
|
|
48
|
-
if (!
|
|
49
|
-
|
|
50
|
-
|
|
129
|
+
const framework = FRAMEWORK_CONFIG_KEYS[key];
|
|
130
|
+
if (!framework) return null;
|
|
131
|
+
const result = framework.schema.safeParse(value);
|
|
132
|
+
if (result.success) return null;
|
|
133
|
+
const explanation = result.error.issues.map((issue) => issue.message).join('\n');
|
|
134
|
+
return `Invalid value '${value}' for '${key}'.\n\n${explanation}\n\n${framework.why}`;
|
|
51
135
|
}
|
|
52
136
|
|
|
53
137
|
/**
|
|
@@ -131,10 +215,14 @@ export async function handleModuleConfigSet(args: string[]): Promise<CommandResu
|
|
|
131
215
|
// Set config value using service (handles primitive and complex types)
|
|
132
216
|
try {
|
|
133
217
|
await setModuleConfigValue(moduleId, key, value);
|
|
218
|
+
const resolved = settlingWatchState(db, moduleId, key);
|
|
134
219
|
|
|
135
220
|
return {
|
|
136
221
|
success: true,
|
|
137
|
-
message:
|
|
222
|
+
message:
|
|
223
|
+
resolved > 0
|
|
224
|
+
? `Set config for ${moduleId}: ${key} (resolved ${resolved} alert(s) — nothing will report on this module again until it is watched)`
|
|
225
|
+
: `Set config for ${moduleId}: ${key}`,
|
|
138
226
|
};
|
|
139
227
|
} catch (error) {
|
|
140
228
|
return {
|
|
@@ -144,6 +232,69 @@ export async function handleModuleConfigSet(args: string[]): Promise<CommandResu
|
|
|
144
232
|
}
|
|
145
233
|
}
|
|
146
234
|
|
|
235
|
+
/**
|
|
236
|
+
* Handle module config unset command
|
|
237
|
+
*
|
|
238
|
+
* Usage: celilo module config unset <module-id> <key>
|
|
239
|
+
*
|
|
240
|
+
* Removing an override is what returns a module to following its manifest.
|
|
241
|
+
* Without it, an operator who once set a cadence could never go back to the
|
|
242
|
+
* author's suggestion, and every later correction would stop reaching them —
|
|
243
|
+
* the exact failure read-time resolution exists to prevent, aimed at the
|
|
244
|
+
* operators who engaged with the feature.
|
|
245
|
+
*
|
|
246
|
+
* @param args - Command arguments
|
|
247
|
+
* @returns Command result
|
|
248
|
+
*/
|
|
249
|
+
export async function handleModuleConfigUnset(args: string[]): Promise<CommandResult> {
|
|
250
|
+
const error = validateRequiredArgs(args, 2);
|
|
251
|
+
if (error) {
|
|
252
|
+
return {
|
|
253
|
+
success: false,
|
|
254
|
+
error: `${error}\n\nUsage: celilo module config unset <module-id> <key>`,
|
|
255
|
+
};
|
|
256
|
+
}
|
|
257
|
+
|
|
258
|
+
const moduleId = getArg(args, 0);
|
|
259
|
+
const key = getArg(args, 1);
|
|
260
|
+
|
|
261
|
+
if (!moduleId || !key) {
|
|
262
|
+
return { success: false, error: 'Module ID and key are required' };
|
|
263
|
+
}
|
|
264
|
+
|
|
265
|
+
const db = getDb();
|
|
266
|
+
|
|
267
|
+
const module = db.select().from(modules).where(eq(modules.id, moduleId)).get();
|
|
268
|
+
if (!module) {
|
|
269
|
+
return { success: false, error: `Module not found: ${moduleId}` };
|
|
270
|
+
}
|
|
271
|
+
|
|
272
|
+
// Unsetting a key that was never set SUCCEEDS. `unset` states a desired end
|
|
273
|
+
// state, and failing on an already-clean one makes it unusable from any
|
|
274
|
+
// script that cannot check first.
|
|
275
|
+
if (!getModuleConfigValue(moduleId, key, db)) {
|
|
276
|
+
return { success: true, message: `No override set for ${moduleId}: ${key}` };
|
|
277
|
+
}
|
|
278
|
+
|
|
279
|
+
deleteModuleConfig(db, moduleId, key);
|
|
280
|
+
settlingWatchState(db, moduleId, key);
|
|
281
|
+
return {
|
|
282
|
+
success: true,
|
|
283
|
+
message: `Unset config for ${moduleId}: ${key} (now follows the module's manifest)`,
|
|
284
|
+
};
|
|
285
|
+
}
|
|
286
|
+
|
|
287
|
+
/**
|
|
288
|
+
* A module that has just stopped being watched may still own live alerts from
|
|
289
|
+
* its last scheduled runs, and nothing will ever report on them again. Resolve
|
|
290
|
+
* them here rather than leaving them firing with no action able to clear them.
|
|
291
|
+
* No-op for every other key.
|
|
292
|
+
*/
|
|
293
|
+
function settlingWatchState(db: ReturnType<typeof getDb>, moduleId: string, key: string): number {
|
|
294
|
+
if (key !== HEALTH_CHECK_INTERVAL_CONFIG_KEY) return 0;
|
|
295
|
+
return reconcileModuleWatchState(db, moduleId, new Date());
|
|
296
|
+
}
|
|
297
|
+
|
|
147
298
|
/**
|
|
148
299
|
* Handle module config get command
|
|
149
300
|
*
|
|
@@ -5,11 +5,123 @@
|
|
|
5
5
|
import { eq } from 'drizzle-orm';
|
|
6
6
|
import { getDb } from '../../db/client';
|
|
7
7
|
import { capabilities, moduleConfigs, modules, secrets } from '../../db/schema';
|
|
8
|
+
import type { ModuleManifest } from '../../manifest/schema';
|
|
9
|
+
import {
|
|
10
|
+
HEALTH_CHECK_INTERVAL_CONFIG_KEY,
|
|
11
|
+
effectiveHealthCheckCadence,
|
|
12
|
+
} from '../../services/alerting/health-cadence';
|
|
13
|
+
import {
|
|
14
|
+
BACKUP_RETENTION_COUNT_CONFIG_KEY,
|
|
15
|
+
BACKUP_RETENTION_MAX_AGE_DAYS_CONFIG_KEY,
|
|
16
|
+
effectiveBackupRetention,
|
|
17
|
+
prunesNothing,
|
|
18
|
+
} from '../../services/backup-retention';
|
|
19
|
+
import {
|
|
20
|
+
BACKUP_SCHEDULE_CONFIG_KEY,
|
|
21
|
+
effectiveBackupSchedule,
|
|
22
|
+
} from '../../services/backup-schedule';
|
|
23
|
+
import { formatCadence } from '../../services/cadence';
|
|
8
24
|
import { getModuleSystems } from '../../services/deployed-systems';
|
|
25
|
+
import { configOverride, parseStoredConfigValue } from '../../services/module-config';
|
|
9
26
|
import { formatPlacementLine, reconcilePlacement } from '../../services/placement-reconcile';
|
|
10
27
|
import { getArg, validateRequiredArgs } from '../parser';
|
|
11
28
|
import type { CommandResult } from '../types';
|
|
12
29
|
|
|
30
|
+
/**
|
|
31
|
+
* PURE (Rule 10.1): the per-module policy block — what celilo will do to this
|
|
32
|
+
* module, and on whose authority.
|
|
33
|
+
*
|
|
34
|
+
* Both halves are shown deliberately. The stored override alone does not tell
|
|
35
|
+
* an operator what they changed it FROM, and the effective value alone does not
|
|
36
|
+
* tell them whether they set it or the module's author did.
|
|
37
|
+
*/
|
|
38
|
+
export function formatCadencePolicy(input: {
|
|
39
|
+
manifest: ModuleManifest;
|
|
40
|
+
configs: Record<string, unknown>;
|
|
41
|
+
}): string {
|
|
42
|
+
const { manifest, configs } = input;
|
|
43
|
+
const lines = ['Policy:'];
|
|
44
|
+
|
|
45
|
+
if (manifest.hooks?.on_backup) {
|
|
46
|
+
const override = configOverride(configs, BACKUP_SCHEDULE_CONFIG_KEY);
|
|
47
|
+
const effective = formatCadence(effectiveBackupSchedule(manifest, override));
|
|
48
|
+
const suggested = manifest.backup?.schedule;
|
|
49
|
+
if (override !== undefined) {
|
|
50
|
+
lines.push(
|
|
51
|
+
` backup cadence: ${effective} (operator override; manifest suggests ${suggested ?? 'nothing'})`,
|
|
52
|
+
);
|
|
53
|
+
} else if (suggested !== undefined) {
|
|
54
|
+
lines.push(` backup cadence: ${effective} (from the manifest)`);
|
|
55
|
+
} else {
|
|
56
|
+
lines.push(` backup cadence: ${effective} (celilo default; nothing declared or set)`);
|
|
57
|
+
}
|
|
58
|
+
lines.push(` backup retention: ${describeRetention(manifest, configs)}`);
|
|
59
|
+
} else {
|
|
60
|
+
lines.push(' backup cadence: not backed up (module declares no on_backup hook)');
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
if (manifest.hooks?.health_check) {
|
|
64
|
+
const override = configOverride(configs, HEALTH_CHECK_INTERVAL_CONFIG_KEY);
|
|
65
|
+
const effective = effectiveHealthCheckCadence(manifest, override);
|
|
66
|
+
const suggested = manifest.hooks.health_check.interval;
|
|
67
|
+
const value = effective === null ? 'not watched (no cadence set)' : formatCadence(effective);
|
|
68
|
+
if (override !== undefined) {
|
|
69
|
+
lines.push(
|
|
70
|
+
` health check: ${value} (operator override; manifest suggests ${suggested ?? 'nothing'})`,
|
|
71
|
+
);
|
|
72
|
+
} else if (suggested !== undefined) {
|
|
73
|
+
lines.push(` health check: ${value} (from the manifest)`);
|
|
74
|
+
} else {
|
|
75
|
+
lines.push(` health check: ${value}`);
|
|
76
|
+
}
|
|
77
|
+
} else {
|
|
78
|
+
lines.push(' health check: not watched (module declares no health_check hook)');
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
return lines.join('\n');
|
|
82
|
+
}
|
|
83
|
+
|
|
84
|
+
/**
|
|
85
|
+
* Retention in one line, per dimension, saying "unbounded" rather than a number
|
|
86
|
+
* wherever nothing bounds it. An operator reading a bound they never set would
|
|
87
|
+
* reasonably assume backups are being deleted — and the reverse assumption,
|
|
88
|
+
* that something is pruning when nothing is, is how a disk fills.
|
|
89
|
+
*/
|
|
90
|
+
function describeRetention(manifest: ModuleManifest, configs: Record<string, unknown>): string {
|
|
91
|
+
const policy = effectiveBackupRetention(manifest, configs);
|
|
92
|
+
if (prunesNothing(policy)) return 'none — every backup is kept';
|
|
93
|
+
|
|
94
|
+
const declared = manifest.backup?.retention;
|
|
95
|
+
const dimension = (
|
|
96
|
+
effective: number,
|
|
97
|
+
override: string | undefined,
|
|
98
|
+
suggested: number | undefined,
|
|
99
|
+
unit: string,
|
|
100
|
+
): string => {
|
|
101
|
+
if (effective === Number.POSITIVE_INFINITY) return `unbounded ${unit}`;
|
|
102
|
+
const source =
|
|
103
|
+
override !== undefined
|
|
104
|
+
? `operator override; manifest suggests ${suggested ?? 'nothing'}`
|
|
105
|
+
: 'from the manifest';
|
|
106
|
+
return `${effective} ${unit} (${source})`;
|
|
107
|
+
};
|
|
108
|
+
|
|
109
|
+
return [
|
|
110
|
+
dimension(
|
|
111
|
+
policy.count,
|
|
112
|
+
configOverride(configs, BACKUP_RETENTION_COUNT_CONFIG_KEY),
|
|
113
|
+
declared?.count,
|
|
114
|
+
'copies',
|
|
115
|
+
),
|
|
116
|
+
dimension(
|
|
117
|
+
policy.maxAgeDays,
|
|
118
|
+
configOverride(configs, BACKUP_RETENTION_MAX_AGE_DAYS_CONFIG_KEY),
|
|
119
|
+
declared?.max_age_days,
|
|
120
|
+
'days',
|
|
121
|
+
),
|
|
122
|
+
].join(', ');
|
|
123
|
+
}
|
|
124
|
+
|
|
13
125
|
/**
|
|
14
126
|
* Handle module status command
|
|
15
127
|
*
|
|
@@ -114,6 +226,18 @@ export async function handleModuleStatus(args: string[]): Promise<CommandResult>
|
|
|
114
226
|
sections.push('Configuration: (none)');
|
|
115
227
|
}
|
|
116
228
|
|
|
229
|
+
// Section 2b: Per-module policy — what celilo will DO to this module, and
|
|
230
|
+
// whether that came from the operator or from the module's author. A raw
|
|
231
|
+
// config key does not tell an operator what the manifest said, and an
|
|
232
|
+
// effective value alone does not tell them whether they are the one who set
|
|
233
|
+
// it. Both, always.
|
|
234
|
+
sections.push(
|
|
235
|
+
formatCadencePolicy({
|
|
236
|
+
manifest: module.manifestData as ModuleManifest,
|
|
237
|
+
configs: Object.fromEntries(configs.map((c) => [c.key, parseStoredConfigValue(c)])),
|
|
238
|
+
}),
|
|
239
|
+
);
|
|
240
|
+
|
|
117
241
|
// Section 3: Secrets
|
|
118
242
|
if (moduleSecrets.length > 0) {
|
|
119
243
|
const secretLines = ['Secrets:'];
|