@celilo/cli 0.23.0 → 0.24.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CELILO_CORE_MODULES.md +2 -2
- package/CELILO_SUBSYSTEMS.md +27 -7
- package/package.json +6 -5
- package/src/cli/commands/alerts-act.ts +1 -1
- package/src/cli/commands/backup-create.ts +26 -11
- package/src/cli/commands/backup-list.test.ts +83 -0
- package/src/cli/commands/backup-list.ts +67 -3
- package/src/cli/commands/backup-prune.ts +17 -17
- package/src/cli/commands/backup-sweep.ts +20 -8
- package/src/cli/commands/firewall-interface-list.test.ts +85 -0
- package/src/cli/commands/firewall-interface-list.ts +123 -0
- package/src/cli/commands/machine-add.ts +30 -2
- package/src/cli/commands/module-config.test.ts +64 -2
- package/src/cli/commands/module-config.ts +159 -8
- package/src/cli/commands/module-status.ts +124 -0
- package/src/cli/commands/monitor.ts +116 -19
- package/src/cli/commands/system-migrate.ts +14 -0
- package/src/cli/commands/system-update.ts +4 -1
- package/src/cli/completion.ts +35 -9
- package/src/cli/index.ts +59 -2
- package/src/cli/tui/audit-state.ts +2 -0
- package/src/hooks/capability-loader.ts +130 -4
- package/src/hooks/types.ts +2 -1
- package/src/manifest/contracts/v1.ts +16 -0
- package/src/manifest/schema.ts +40 -65
- package/src/services/alerting/builtin-monitors.test.ts +18 -10
- package/src/services/alerting/cadence-migration.test.ts +155 -0
- package/src/services/alerting/cadence-migration.ts +90 -0
- package/src/services/alerting/coverage-source.ts +8 -11
- package/src/services/alerting/deploy-hooks.test.ts +16 -7
- package/src/services/alerting/deploy-hooks.ts +11 -5
- package/src/services/alerting/health-cadence.test.ts +58 -0
- package/src/services/alerting/health-cadence.ts +128 -0
- package/src/services/alerting/health-coverage.ts +18 -8
- package/src/services/alerting/monitors.ts +50 -15
- package/src/services/alerting/sweep-runner.test.ts +51 -3
- package/src/services/alerting/sweep-runner.ts +30 -7
- package/src/services/audit/backup-source.ts +24 -1
- package/src/services/audit/backups.test.ts +95 -10
- package/src/services/audit/backups.ts +40 -37
- package/src/services/audit/interface-classification.test.ts +220 -0
- package/src/services/audit/interface-classification.ts +167 -0
- package/src/services/audit/types.ts +2 -1
- package/src/services/backup-age-agreement.test.ts +118 -0
- package/src/services/backup-create.ts +36 -30
- package/src/services/backup-metadata.ts +52 -1
- package/src/services/backup-retention.test.ts +123 -0
- package/src/services/backup-retention.ts +66 -5
- package/src/services/backup-schedule.test.ts +166 -0
- package/src/services/backup-schedule.ts +105 -15
- package/src/services/backup-staging.ts +14 -1
- package/src/services/backup-sweep.test.ts +22 -3
- package/src/services/backup-sweep.ts +15 -5
- package/src/services/cadence.test.ts +97 -0
- package/src/services/cadence.ts +165 -0
- package/src/services/machine-detector.ts +23 -1
- package/src/services/module-config.ts +33 -0
- package/src/services/storage-providers/s3.test.ts +96 -13
- package/src/services/storage-providers/s3.ts +48 -15
- package/src/services/zone-detector.test.ts +34 -3
- package/src/services/zone-detector.ts +33 -13
|
@@ -154,6 +154,32 @@ export async function resolveFirewallNatIp(db: DbClient): Promise<string | undef
|
|
|
154
154
|
return undefined;
|
|
155
155
|
}
|
|
156
156
|
|
|
157
|
+
/**
|
|
158
|
+
* The addresses of every machine a `firewall` capability provider manages.
|
|
159
|
+
*
|
|
160
|
+
* This is the authoritative answer to "which machines are firewalls", and it has
|
|
161
|
+
* to be: a machine's stored `role` is decided by `machine add`, from the zones
|
|
162
|
+
* declared AT THAT MOMENT. The normal order is `machine add` and THEN deploy
|
|
163
|
+
* iptables, whose `on_install` writes the zone subnets — so the firewall is
|
|
164
|
+
* recorded as a plain host and stays that way. A firewall provider naming an IP
|
|
165
|
+
* is a statement, not an inference from a snapshot.
|
|
166
|
+
*/
|
|
167
|
+
export async function listFirewallIps(db: DbClient): Promise<string[]> {
|
|
168
|
+
const providers = db
|
|
169
|
+
.select()
|
|
170
|
+
.from(capabilities)
|
|
171
|
+
.where(eq(capabilities.capabilityName, 'firewall'))
|
|
172
|
+
.all();
|
|
173
|
+
const ips: string[] = [];
|
|
174
|
+
for (const provider of providers) {
|
|
175
|
+
const config = await loadModuleConfig(provider.moduleId, db);
|
|
176
|
+
if (typeof config.firewall_ip === 'string' && config.firewall_ip) {
|
|
177
|
+
ips.push(config.firewall_ip);
|
|
178
|
+
}
|
|
179
|
+
}
|
|
180
|
+
return ips;
|
|
181
|
+
}
|
|
182
|
+
|
|
157
183
|
/**
|
|
158
184
|
* Caddy's zone-routable IP — its own DMZ ingress address (`target_ip`, the same
|
|
159
185
|
* value public_web exposes as `dmz_ip`). This is the in-zone split-horizon
|
|
@@ -606,6 +632,9 @@ function buildCapabilityInterface(
|
|
|
606
632
|
undefined, // no upstream — the chain path handles that
|
|
607
633
|
undefined, // logger is applied by wrapWithLogging at the loader site
|
|
608
634
|
trustedSourceStore,
|
|
635
|
+
// LIVE, for the same reason as the chain path: a declaration written
|
|
636
|
+
// during this hook run must be visible to the converge that follows it.
|
|
637
|
+
zones ? { list: zones.declaredNetworks } : undefined,
|
|
609
638
|
);
|
|
610
639
|
return stampProvider(iface, providerModuleId);
|
|
611
640
|
}
|
|
@@ -707,6 +736,18 @@ interface FirewallZones {
|
|
|
707
736
|
trustedSubnets: string[];
|
|
708
737
|
/** celilo's control-plane network, as a DESTINATION for trusted sources. */
|
|
709
738
|
controlPlaneSubnet?: string;
|
|
739
|
+
/**
|
|
740
|
+
* Every zone with a declared subnet, WITH its name — the input to interface
|
|
741
|
+
* classification. Excludes `external`, which is a residual rather than a
|
|
742
|
+
* subnet (design D1, amended).
|
|
743
|
+
*/
|
|
744
|
+
/**
|
|
745
|
+
* A LIVE read, not a snapshot. Everything else here is captured when the
|
|
746
|
+
* capability is built — before any hook runs — and for declarations that is
|
|
747
|
+
* wrong: `wireguard` declares its VPN subnet and brings `wg0` up inside one
|
|
748
|
+
* hook run, and the converge that follows has to see the declaration.
|
|
749
|
+
*/
|
|
750
|
+
declaredNetworks: () => Array<{ zone: string; subnet: string }>;
|
|
710
751
|
/**
|
|
711
752
|
* Every declared zone subnet — read from
|
|
712
753
|
* `network.<zone>.subnet` across all of `NETWORK_ZONES`, not just the three
|
|
@@ -806,6 +847,35 @@ export function loadTrustedSubnets(db: DbClient, firewallIp?: string): TrustedSu
|
|
|
806
847
|
* of them. Zones with no configured subnet are omitted — their traffic stays
|
|
807
848
|
* denied (fail-closed).
|
|
808
849
|
*/
|
|
850
|
+
/**
|
|
851
|
+
* Every network celilo can attribute an interface to: one entry per
|
|
852
|
+
* `network.<name>.subnet` in system config.
|
|
853
|
+
*
|
|
854
|
+
* Read from the config rather than from `NETWORK_ZONES`, because celilo declares
|
|
855
|
+
* networks that are not placement zones. `wireguard`'s `on_install` writes
|
|
856
|
+
* `network.control-plane-vpn.subnet` before it brings `wg0` up, exactly as the
|
|
857
|
+
* design requires (D3), and the firewall is supposed to classify `wg0` "by the
|
|
858
|
+
* same subnet containment it uses for every other leg". Reading only
|
|
859
|
+
* `NETWORK_ZONES` left that declaration invisible: the converge could not
|
|
860
|
+
* attribute `wg0` and refused — and on a firewall with a baseline it would have
|
|
861
|
+
* ISOLATED it, shutting down the admin VPN.
|
|
862
|
+
*
|
|
863
|
+
* `external` is deliberately absent even when something has set
|
|
864
|
+
* `network.external.subnet`: it is the RESIDUAL, decided by `isPubliclyRoutable`
|
|
865
|
+
* and never by containment (design D1, amended). Matching an interface to
|
|
866
|
+
* `external` by subnet would reintroduce the overload this change removes.
|
|
867
|
+
*/
|
|
868
|
+
export function readDeclaredNetworks(db: DbClient): Array<{ zone: string; subnet: string }> {
|
|
869
|
+
return db
|
|
870
|
+
.select()
|
|
871
|
+
.from(systemConfig)
|
|
872
|
+
.all()
|
|
873
|
+
.flatMap((row) => {
|
|
874
|
+
const zone = /^network\.(.+)\.subnet$/.exec(row.key)?.[1];
|
|
875
|
+
return zone && zone !== 'external' && row.value ? [{ zone, subnet: row.value }] : [];
|
|
876
|
+
});
|
|
877
|
+
}
|
|
878
|
+
|
|
809
879
|
function loadFirewallZones(db: DbClient): FirewallZones {
|
|
810
880
|
const zoneTiers: Array<{ name: string; subnet: string }> = [];
|
|
811
881
|
for (const zone of ZONE_TIER_ORDER) {
|
|
@@ -822,17 +892,65 @@ function loadFirewallZones(db: DbClient): FirewallZones {
|
|
|
822
892
|
// snapshot put the subnet straight back, so the set could grow but never
|
|
823
893
|
// shrink. That is exactly what happened: `on_uninstall` withdrew the VPN's
|
|
824
894
|
// trusted source, logged success, and the reach rules were re-rendered anyway.
|
|
895
|
+
// Every zone that HAS a declared subnet, keeping the zone NAME. Interface
|
|
896
|
+
// classification needs the name — `external` and `internal` are the only two
|
|
897
|
+
// legs a default route may leave through (design D11), which is not a question
|
|
898
|
+
// a bare list of subnets can answer.
|
|
899
|
+
//
|
|
900
|
+
// `external` is deliberately absent from this list even when something has set
|
|
901
|
+
// `network.external.subnet`: `external` is the RESIDUAL, decided by
|
|
902
|
+
// `isPubliclyRoutable`, never by containment (design D1, amended). Matching an
|
|
903
|
+
// interface to `external` by subnet would reintroduce the overload this change
|
|
904
|
+
// removes.
|
|
905
|
+
//
|
|
906
|
+
// EVERY declared network, not just the six `NETWORK_ZONES`. celilo declares
|
|
907
|
+
// networks that are not placement zones — `network.control-plane-vpn.subnet`,
|
|
908
|
+
// written by `wireguard`'s `on_install` — and the design is explicit that the
|
|
909
|
+
// firewall "classifies `wg0` by the same subnet containment it uses for every
|
|
910
|
+
// other leg". Reading only `NETWORK_ZONES` left that declaration invisible:
|
|
911
|
+
// the module declared the subnet before bringing the interface up, exactly as
|
|
912
|
+
// designed, and the converge still could not attribute `wg0` and refused. On a
|
|
913
|
+
// firewall that already had a baseline it would have gone further and ISOLATED
|
|
914
|
+
// the interface — celilo shutting down the admin VPN, which for a remote
|
|
915
|
+
// operator is the way back in.
|
|
916
|
+
//
|
|
917
|
+
// So the source is the config itself. A hardcoded list here could only ever
|
|
918
|
+
// describe the networks celilo knew about when this line was written.
|
|
919
|
+
|
|
825
920
|
return {
|
|
826
921
|
zoneTiers,
|
|
827
922
|
trustedSubnets: loadTrustedSubnets(db).map((e) => e.subnet),
|
|
828
923
|
controlPlaneSubnet: loadControlPlaneSubnet(db) ?? readZoneSubnet(db, 'internal'),
|
|
829
|
-
|
|
830
|
-
|
|
831
|
-
|
|
832
|
-
|
|
924
|
+
declaredNetworks: () => readDeclaredNetworks(db),
|
|
925
|
+
// Placement ZONES only — deliberately NOT `declaredNetworks`, which is wider.
|
|
926
|
+
// Fronting a subnet renders translation for it; the control-plane VPN
|
|
927
|
+
// reaches the fleet as a trusted source instead, and giving it a second
|
|
928
|
+
// mechanism would change the rendered ruleset for every existing fleet.
|
|
929
|
+
frontedSubnets: NETWORK_ZONES.filter((zone) => zone !== 'external').flatMap((zone) => {
|
|
930
|
+
const subnet = readZoneSubnet(db, zone);
|
|
931
|
+
return subnet ? [subnet] : [];
|
|
932
|
+
}),
|
|
833
933
|
};
|
|
834
934
|
}
|
|
835
935
|
|
|
936
|
+
/**
|
|
937
|
+
* The interface baseline as stored in module config: a comma-separated list of
|
|
938
|
+
* interface names, or absent.
|
|
939
|
+
*
|
|
940
|
+
* Absent and empty are the SAME thing here and both mean "no baseline" — a
|
|
941
|
+
* firewall with zero interfaces does not exist, so an empty string can only be
|
|
942
|
+
* a cleared or never-written value. Treating it as a baseline of nothing would
|
|
943
|
+
* mean every interface is "new", which is the opposite of what it says.
|
|
944
|
+
*/
|
|
945
|
+
function parseInterfaceBaseline(raw: unknown): string[] | undefined {
|
|
946
|
+
if (typeof raw !== 'string') return undefined;
|
|
947
|
+
const names = raw
|
|
948
|
+
.split(',')
|
|
949
|
+
.map((n) => n.trim())
|
|
950
|
+
.filter((n) => n.length > 0);
|
|
951
|
+
return names.length > 0 ? names : undefined;
|
|
952
|
+
}
|
|
953
|
+
|
|
836
954
|
async function buildFirewallChain(
|
|
837
955
|
allProviders: Array<{
|
|
838
956
|
id: number;
|
|
@@ -982,11 +1100,19 @@ async function buildFirewallChain(
|
|
|
982
1100
|
trustedSubnets: zones.trustedSubnets,
|
|
983
1101
|
controlPlaneSubnet: zones.controlPlaneSubnet,
|
|
984
1102
|
frontedSubnets: zones.frontedSubnets,
|
|
1103
|
+
// The recorded baseline (D12), from this firewall's own module config.
|
|
1104
|
+
// Absent means the box has never converged cleanly, so an interface
|
|
1105
|
+
// celilo cannot attribute refuses rather than being disabled.
|
|
1106
|
+
interfaceBaseline: parseInterfaceBaseline(provConfig.interface_baseline),
|
|
985
1107
|
},
|
|
986
1108
|
store,
|
|
987
1109
|
currentUpstream,
|
|
988
1110
|
logger,
|
|
989
1111
|
trustedSourceStore,
|
|
1112
|
+
// LIVE, not a snapshot: a declaration written during this hook run —
|
|
1113
|
+
// `wireguard` declaring its VPN subnet before creating `wg0` — must be
|
|
1114
|
+
// visible to the converge that follows it in the same run.
|
|
1115
|
+
{ list: zones.declaredNetworks },
|
|
990
1116
|
);
|
|
991
1117
|
|
|
992
1118
|
debugLog(`firewall chain: wired ${provider.moduleId} → ${hasExternal.moduleId}`);
|
package/src/hooks/types.ts
CHANGED
|
@@ -120,7 +120,8 @@ export type HookName =
|
|
|
120
120
|
| 'on_backup_analyze'
|
|
121
121
|
| 'on_restore'
|
|
122
122
|
| 'on_system_event'
|
|
123
|
-
| 'refresh_registrations'
|
|
123
|
+
| 'refresh_registrations'
|
|
124
|
+
| 'reassert_dhcp_dns';
|
|
124
125
|
|
|
125
126
|
/**
|
|
126
127
|
* Hook manifest section - maps hook names to definitions
|
|
@@ -194,6 +194,22 @@ export const V1_HOOKS: ContractHooks = {
|
|
|
194
194
|
},
|
|
195
195
|
outputs: {},
|
|
196
196
|
},
|
|
197
|
+
/**
|
|
198
|
+
* Periodic re-assertion of the resolver a dns_internal provider hands
|
|
199
|
+
* out over DHCP (celilo#739). Some routers regenerate that value from
|
|
200
|
+
* their own upstream list on a timer and silently undo what on_install
|
|
201
|
+
* set — the write succeeds and reads back correct, so only a client
|
|
202
|
+
* renewing later ever sees the wrong resolver. No framework-injected
|
|
203
|
+
* inputs: the hook derives the address it wants from its own config and
|
|
204
|
+
* systems, and reads the device before writing. Typically driven by a
|
|
205
|
+
* `timer.tick.1m` subscription, because the tick interval is the
|
|
206
|
+
* worst-case window in which a renewing client can be handed the wrong
|
|
207
|
+
* resolver for a full lease.
|
|
208
|
+
*/
|
|
209
|
+
reassert_dhcp_dns: {
|
|
210
|
+
inputs: {},
|
|
211
|
+
outputs: {},
|
|
212
|
+
},
|
|
197
213
|
/**
|
|
198
214
|
* Build-bus upstream publish hook. The executor passes the
|
|
199
215
|
* PublishEvent fields as env vars (CELILO_EVENT_PAYLOAD,
|
package/src/manifest/schema.ts
CHANGED
|
@@ -1,5 +1,10 @@
|
|
|
1
1
|
import { z } from 'zod';
|
|
2
2
|
import { NETWORK_ZONES } from '../db/schema';
|
|
3
|
+
import {
|
|
4
|
+
BACKUP_CADENCE_FLOOR_MINUTES,
|
|
5
|
+
MONITOR_INTERVAL_FLOOR_MINUTES,
|
|
6
|
+
cadenceSchema,
|
|
7
|
+
} from '../services/cadence';
|
|
3
8
|
import { SUPPORTED_CONTRACT_VERSIONS } from './contracts';
|
|
4
9
|
|
|
5
10
|
/**
|
|
@@ -273,73 +278,26 @@ export const LifecycleHookSchema = z.object({
|
|
|
273
278
|
timeout: z.number().positive().optional(),
|
|
274
279
|
});
|
|
275
280
|
|
|
276
|
-
/**
|
|
277
|
-
* Floor for a module's suggested `health_check` interval.
|
|
278
|
-
*
|
|
279
|
-
* The monitor sweep rides the event bus's fixed `timer.tick.5m`
|
|
280
|
-
* (packages/event-bus/src/timer.ts), so an interval below five minutes
|
|
281
|
-
* cannot be honoured — it would silently round up. Rejecting it at manifest
|
|
282
|
-
* validation is better than accepting a promise celilo can't keep.
|
|
283
|
-
*/
|
|
284
|
-
export const MONITOR_INTERVAL_FLOOR_MINUTES = 5;
|
|
285
|
-
|
|
286
|
-
const DURATION_PATTERN = /^(\d+)(m|h|d)$/;
|
|
287
|
-
|
|
288
|
-
/**
|
|
289
|
-
* Parse a duration string (`15m`, `1h`, `1d`) to whole minutes.
|
|
290
|
-
* Returns null when the string is not a well-formed duration.
|
|
291
|
-
*/
|
|
292
|
-
export function parseIntervalMinutes(value: string): number | null {
|
|
293
|
-
const match = DURATION_PATTERN.exec(value);
|
|
294
|
-
if (!match) return null;
|
|
295
|
-
const amount = Number.parseInt(match[1], 10);
|
|
296
|
-
if (!Number.isFinite(amount) || amount <= 0) return null;
|
|
297
|
-
const unit = match[2];
|
|
298
|
-
if (unit === 'm') return amount;
|
|
299
|
-
if (unit === 'h') return amount * 60;
|
|
300
|
-
return amount * 60 * 24;
|
|
301
|
-
}
|
|
302
|
-
|
|
303
281
|
/**
|
|
304
282
|
* `health_check` accepts everything a lifecycle hook does, plus an optional
|
|
305
283
|
* `interval` — the module author's SUGGESTED monitoring cadence.
|
|
306
284
|
*
|
|
307
|
-
* It is only a suggestion: the operator's `
|
|
308
|
-
*
|
|
309
|
-
*
|
|
310
|
-
*
|
|
285
|
+
* It is only a suggestion: the operator's `health_check_interval` override
|
|
286
|
+
* decides, and resolution happens at read time so a corrected suggestion
|
|
287
|
+
* reaches every install that has not overridden
|
|
288
|
+
* ([[services/alerting/health-cadence.ts]]). A module that omits it is not
|
|
289
|
+
* monitored until someone names a cadence.
|
|
290
|
+
*
|
|
291
|
+
* The same spellings as every other cadence in celilo — a named period, a
|
|
292
|
+
* duration, or `manual` — with the floor derived from the alerting sweep's
|
|
293
|
+
* tick.
|
|
311
294
|
*/
|
|
312
295
|
export const HealthCheckHookSchema = LifecycleHookSchema.extend({
|
|
313
|
-
|
|
314
|
-
|
|
315
|
-
|
|
316
|
-
|
|
317
|
-
|
|
318
|
-
// sharing DURATION_PATTERN.
|
|
319
|
-
interval: z
|
|
320
|
-
.string()
|
|
321
|
-
.regex(DURATION_PATTERN)
|
|
322
|
-
.optional()
|
|
323
|
-
.describe(
|
|
324
|
-
'Suggested monitoring cadence, e.g. "15m", "1h", "1d". Must be 5m or longer — the monitor sweep runs on a 5-minute grid.',
|
|
325
|
-
)
|
|
326
|
-
.superRefine((value, ctx) => {
|
|
327
|
-
if (value === undefined) return;
|
|
328
|
-
const minutes = parseIntervalMinutes(value);
|
|
329
|
-
if (minutes === null) {
|
|
330
|
-
ctx.addIssue({
|
|
331
|
-
code: z.ZodIssueCode.custom,
|
|
332
|
-
message: `Invalid health_check interval "${value}". Use a duration like "15m", "1h", or "1d".`,
|
|
333
|
-
});
|
|
334
|
-
return;
|
|
335
|
-
}
|
|
336
|
-
if (minutes < MONITOR_INTERVAL_FLOOR_MINUTES) {
|
|
337
|
-
ctx.addIssue({
|
|
338
|
-
code: z.ZodIssueCode.custom,
|
|
339
|
-
message: `health_check interval "${value}" is below the ${MONITOR_INTERVAL_FLOOR_MINUTES}m floor — the monitor sweep runs on a ${MONITOR_INTERVAL_FLOOR_MINUTES}-minute grid and cannot run it more often. Use "${MONITOR_INTERVAL_FLOOR_MINUTES}m" or longer.`,
|
|
340
|
-
});
|
|
341
|
-
}
|
|
342
|
-
}),
|
|
296
|
+
interval: cadenceSchema({
|
|
297
|
+
floorMinutes: MONITOR_INTERVAL_FLOOR_MINUTES,
|
|
298
|
+
description:
|
|
299
|
+
'Suggested monitoring cadence: a duration like "15m", "1h" or "1d", a named period (hourly, daily, weekly, monthly), or "manual". The monitor sweep runs on a 5-minute grid, so nothing finer than 5m can be served.',
|
|
300
|
+
}).optional(),
|
|
343
301
|
});
|
|
344
302
|
|
|
345
303
|
/**
|
|
@@ -639,6 +597,16 @@ export const ModuleManifestSchema = z
|
|
|
639
597
|
* designs/DISPATCHER_DAEMON_AND_TIMER_EVENTS.md (B3).
|
|
640
598
|
*/
|
|
641
599
|
refresh_registrations: LifecycleHookSchema.optional(),
|
|
600
|
+
/**
|
|
601
|
+
* Periodic re-assertion of the resolver a dns_internal provider
|
|
602
|
+
* hands out over DHCP. Some routers regenerate that value from
|
|
603
|
+
* their own upstream list on a timer, silently undoing what
|
|
604
|
+
* on_install set. The hook reads the device before writing, so a
|
|
605
|
+
* quiet minute costs one query. Subscribe it to `timer.tick.1m` —
|
|
606
|
+
* the tick interval IS the worst-case window in which a renewing
|
|
607
|
+
* client can be handed the wrong resolver. See celilo#739.
|
|
608
|
+
*/
|
|
609
|
+
reassert_dhcp_dns: LifecycleHookSchema.optional(),
|
|
642
610
|
/**
|
|
643
611
|
* Build-bus upstream publish hooks. Array (a module can react
|
|
644
612
|
* to multiple upstream packages with different actions). See
|
|
@@ -694,10 +662,17 @@ export const ModuleManifestSchema = z
|
|
|
694
662
|
|
|
695
663
|
backup: z
|
|
696
664
|
.object({
|
|
697
|
-
//
|
|
698
|
-
//
|
|
699
|
-
// [[services/backup-schedule.ts]].
|
|
700
|
-
|
|
665
|
+
// The author's SUGGESTED cadence. The operator's `backup_schedule`
|
|
666
|
+
// override wins and is resolved at read time — see
|
|
667
|
+
// [[services/backup-schedule.ts]]. Absent from both means `daily`;
|
|
668
|
+
// opting out of backups entirely is a real decision and takes an
|
|
669
|
+
// explicit `manual`.
|
|
670
|
+
//
|
|
671
|
+
schedule: cadenceSchema({
|
|
672
|
+
floorMinutes: BACKUP_CADENCE_FLOOR_MINUTES,
|
|
673
|
+
description:
|
|
674
|
+
'Suggested backup cadence: a named period (hourly, daily, weekly, monthly), a duration like "6h", or "manual" to opt out. The backup sweep runs hourly, so nothing finer than 1h can be served.',
|
|
675
|
+
}).default('daily'),
|
|
701
676
|
retention: z
|
|
702
677
|
.object({
|
|
703
678
|
count: z.number().int().positive().default(7),
|
|
@@ -116,7 +116,7 @@ describe('failingKeysFromFindings', () => {
|
|
|
116
116
|
});
|
|
117
117
|
|
|
118
118
|
describe('healthCoverageFailingKeys', () => {
|
|
119
|
-
const base = { hasHealthCheckHook: true,
|
|
119
|
+
const base = { hasHealthCheckHook: true, cadence: { minutes: 15 } } as const;
|
|
120
120
|
|
|
121
121
|
test('a monitored module produces no finding', () => {
|
|
122
122
|
expect(healthCoverageFailingKeys([{ id: 'caddy', state: 'VERIFIED', ...base }])).toEqual([]);
|
|
@@ -124,16 +124,26 @@ describe('healthCoverageFailingKeys', () => {
|
|
|
124
124
|
|
|
125
125
|
test('a module with no health_check hook is surfaced', () => {
|
|
126
126
|
const keys = healthCoverageFailingKeys([
|
|
127
|
-
{ id: 'signal', state: 'INSTALLED', hasHealthCheckHook: false,
|
|
127
|
+
{ id: 'signal', state: 'INSTALLED', hasHealthCheckHook: false, cadence: null },
|
|
128
128
|
]);
|
|
129
129
|
expect(keys).toHaveLength(1);
|
|
130
130
|
expect(keys[0].key).toBe('builtin:health_coverage/module:signal');
|
|
131
131
|
expect(keys[0].message).toContain('no health_check hook');
|
|
132
132
|
});
|
|
133
133
|
|
|
134
|
-
test('
|
|
134
|
+
test('an operator opt-out raises no finding — it is a decision, not a gap', () => {
|
|
135
|
+
// The finding would ask for the action they just declined, so nothing they
|
|
136
|
+
// could do would ever clear it.
|
|
137
|
+
expect(
|
|
138
|
+
healthCoverageFailingKeys([
|
|
139
|
+
{ id: 'lunacycle', state: 'VERIFIED', hasHealthCheckHook: true, cadence: 'manual' },
|
|
140
|
+
]),
|
|
141
|
+
).toEqual([]);
|
|
142
|
+
});
|
|
143
|
+
|
|
144
|
+
test('a module with a hook but no cadence is surfaced differently', () => {
|
|
135
145
|
const keys = healthCoverageFailingKeys([
|
|
136
|
-
{ id: 'caddy', state: 'VERIFIED', hasHealthCheckHook: true,
|
|
146
|
+
{ id: 'caddy', state: 'VERIFIED', hasHealthCheckHook: true, cadence: null },
|
|
137
147
|
]);
|
|
138
148
|
expect(keys).toHaveLength(1);
|
|
139
149
|
expect(keys[0].message).toContain('nothing schedules it');
|
|
@@ -142,7 +152,7 @@ describe('healthCoverageFailingKeys', () => {
|
|
|
142
152
|
// A coverage gap is real but is not an outage — it must never page.
|
|
143
153
|
test('findings are warning severity', () => {
|
|
144
154
|
const keys = healthCoverageFailingKeys([
|
|
145
|
-
{ id: 'caddy', state: 'VERIFIED', hasHealthCheckHook: false,
|
|
155
|
+
{ id: 'caddy', state: 'VERIFIED', hasHealthCheckHook: false, cadence: null },
|
|
146
156
|
]);
|
|
147
157
|
expect(keys[0].severity).toBe('warning');
|
|
148
158
|
});
|
|
@@ -153,9 +163,7 @@ describe('healthCoverageFailingKeys', () => {
|
|
|
153
163
|
'ignores a module in state %s',
|
|
154
164
|
(state) => {
|
|
155
165
|
expect(
|
|
156
|
-
healthCoverageFailingKeys([
|
|
157
|
-
{ id: 'x', state, hasHealthCheckHook: false, hasEnabledMonitor: false },
|
|
158
|
-
]),
|
|
166
|
+
healthCoverageFailingKeys([{ id: 'x', state, hasHealthCheckHook: false, cadence: null }]),
|
|
159
167
|
).toEqual([]);
|
|
160
168
|
},
|
|
161
169
|
);
|
|
@@ -163,8 +171,8 @@ describe('healthCoverageFailingKeys', () => {
|
|
|
163
171
|
test('reports only the modules that are actually uncovered', () => {
|
|
164
172
|
const keys = healthCoverageFailingKeys([
|
|
165
173
|
{ id: 'caddy', state: 'VERIFIED', ...base },
|
|
166
|
-
{ id: 'signal', state: 'INSTALLED', hasHealthCheckHook: false,
|
|
167
|
-
{ id: 'forgejo', state: 'INSTALLED', hasHealthCheckHook: true,
|
|
174
|
+
{ id: 'signal', state: 'INSTALLED', hasHealthCheckHook: false, cadence: null },
|
|
175
|
+
{ id: 'forgejo', state: 'INSTALLED', hasHealthCheckHook: true, cadence: null },
|
|
168
176
|
]);
|
|
169
177
|
expect(keys.map((k) => k.key)).toEqual([
|
|
170
178
|
'builtin:health_coverage/module:signal',
|
|
@@ -0,0 +1,155 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The migrate step, and the invariant it protects.
|
|
3
|
+
*
|
|
4
|
+
* A `module_hook` monitor's stored `intervalMinutes` and `enabled` are no
|
|
5
|
+
* longer read: cadence resolves from `module_configs`. That is only safe on an
|
|
6
|
+
* existing fleet because `celilo system migrate` — which the `.deb` postinst
|
|
7
|
+
* runs on every apt upgrade — carries the diverging rows over first. Without
|
|
8
|
+
* it, the upgrade that shipped this change would silently revert every
|
|
9
|
+
* operator's hand-set cadence and resume watching modules they had disabled.
|
|
10
|
+
*/
|
|
11
|
+
|
|
12
|
+
import { afterEach, beforeEach, describe, expect, test } from 'bun:test';
|
|
13
|
+
import { mkdtempSync, rmSync } from 'node:fs';
|
|
14
|
+
import { tmpdir } from 'node:os';
|
|
15
|
+
import { join } from 'node:path';
|
|
16
|
+
import { closeDb, getDb } from '../../db/client';
|
|
17
|
+
import { runMigrations } from '../../db/migrate';
|
|
18
|
+
import { modules, monitors } from '../../db/schema';
|
|
19
|
+
import { getModuleConfigValue, upsertModuleConfig } from '../module-config';
|
|
20
|
+
import { migrateMonitorCadences } from './cadence-migration';
|
|
21
|
+
import { HEALTH_CHECK_INTERVAL_CONFIG_KEY, loadModuleHealthCadences } from './health-cadence';
|
|
22
|
+
|
|
23
|
+
function addModule(id: string, interval?: string): void {
|
|
24
|
+
getDb()
|
|
25
|
+
.insert(modules)
|
|
26
|
+
.values({
|
|
27
|
+
id,
|
|
28
|
+
name: id,
|
|
29
|
+
sourcePath: `/tmp/${id}`,
|
|
30
|
+
version: '1.0.0',
|
|
31
|
+
state: 'INSTALLED',
|
|
32
|
+
manifestData: {
|
|
33
|
+
id,
|
|
34
|
+
hooks: { health_check: { script: './health.ts', ...(interval ? { interval } : {}) } },
|
|
35
|
+
},
|
|
36
|
+
})
|
|
37
|
+
.run();
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
function addMonitor(target: string, intervalMinutes: number, enabled = true): void {
|
|
41
|
+
getDb()
|
|
42
|
+
.insert(monitors)
|
|
43
|
+
.values({
|
|
44
|
+
id: `monitor-${target}`,
|
|
45
|
+
kind: 'module_hook',
|
|
46
|
+
target,
|
|
47
|
+
intervalMinutes,
|
|
48
|
+
enabled,
|
|
49
|
+
})
|
|
50
|
+
.run();
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
function override(moduleId: string): string | undefined {
|
|
54
|
+
const row = getModuleConfigValue(moduleId, HEALTH_CHECK_INTERVAL_CONFIG_KEY, getDb());
|
|
55
|
+
return row === null ? undefined : String(row.value);
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
describe('migrateMonitorCadences', () => {
|
|
59
|
+
let dir: string;
|
|
60
|
+
|
|
61
|
+
beforeEach(async () => {
|
|
62
|
+
dir = mkdtempSync(join(tmpdir(), 'celilo-cadence-migration-'));
|
|
63
|
+
process.env.CELILO_DB_PATH = join(dir, 'celilo.db');
|
|
64
|
+
await runMigrations(process.env.CELILO_DB_PATH);
|
|
65
|
+
});
|
|
66
|
+
|
|
67
|
+
afterEach(() => {
|
|
68
|
+
closeDb();
|
|
69
|
+
process.env.CELILO_DB_PATH = undefined;
|
|
70
|
+
rmSync(dir, { recursive: true, force: true });
|
|
71
|
+
});
|
|
72
|
+
|
|
73
|
+
test("a cadence that diverges from the manifest is carried over, so the upgrade can't revert it", () => {
|
|
74
|
+
addModule('caddy', '15m');
|
|
75
|
+
addMonitor('caddy', 60); // an operator re-cadenced this to hourly
|
|
76
|
+
|
|
77
|
+
const report = migrateMonitorCadences(getDb());
|
|
78
|
+
|
|
79
|
+
expect(override('caddy')).toBe('hourly');
|
|
80
|
+
expect(report.written.get('caddy')).toBe('hourly');
|
|
81
|
+
});
|
|
82
|
+
|
|
83
|
+
test('a disabled monitor becomes manual, so a module nobody wanted watched stays unwatched', () => {
|
|
84
|
+
addModule('lunacycle', '15m');
|
|
85
|
+
addMonitor('lunacycle', 15, false);
|
|
86
|
+
|
|
87
|
+
migrateMonitorCadences(getDb());
|
|
88
|
+
|
|
89
|
+
expect(override('lunacycle')).toBe('manual');
|
|
90
|
+
});
|
|
91
|
+
|
|
92
|
+
test('a cadence matching the manifest writes nothing — it must stay free to follow corrections', () => {
|
|
93
|
+
addModule('forgejo', '15m');
|
|
94
|
+
addMonitor('forgejo', 15);
|
|
95
|
+
|
|
96
|
+
const report = migrateMonitorCadences(getDb());
|
|
97
|
+
|
|
98
|
+
expect(override('forgejo')).toBeUndefined();
|
|
99
|
+
expect(report.unchanged).toContain('forgejo');
|
|
100
|
+
});
|
|
101
|
+
|
|
102
|
+
test('an existing override is never overwritten', () => {
|
|
103
|
+
addModule('authentik', '15m');
|
|
104
|
+
addMonitor('authentik', 60);
|
|
105
|
+
upsertModuleConfig(getDb(), 'authentik', HEALTH_CHECK_INTERVAL_CONFIG_KEY, 'daily');
|
|
106
|
+
|
|
107
|
+
migrateMonitorCadences(getDb());
|
|
108
|
+
|
|
109
|
+
expect(override('authentik')).toBe('daily');
|
|
110
|
+
});
|
|
111
|
+
|
|
112
|
+
test('a second run changes nothing', () => {
|
|
113
|
+
addModule('caddy', '15m');
|
|
114
|
+
addMonitor('caddy', 60);
|
|
115
|
+
|
|
116
|
+
migrateMonitorCadences(getDb());
|
|
117
|
+
const second = migrateMonitorCadences(getDb());
|
|
118
|
+
|
|
119
|
+
expect(second.written.size).toBe(0);
|
|
120
|
+
expect(override('caddy')).toBe('hourly');
|
|
121
|
+
});
|
|
122
|
+
});
|
|
123
|
+
|
|
124
|
+
describe("a module_hook row's stored cadence is not consulted", () => {
|
|
125
|
+
let dir: string;
|
|
126
|
+
|
|
127
|
+
beforeEach(async () => {
|
|
128
|
+
dir = mkdtempSync(join(tmpdir(), 'celilo-cadence-row-'));
|
|
129
|
+
process.env.CELILO_DB_PATH = join(dir, 'celilo.db');
|
|
130
|
+
await runMigrations(process.env.CELILO_DB_PATH);
|
|
131
|
+
});
|
|
132
|
+
|
|
133
|
+
afterEach(() => {
|
|
134
|
+
closeDb();
|
|
135
|
+
process.env.CELILO_DB_PATH = undefined;
|
|
136
|
+
rmSync(dir, { recursive: true, force: true });
|
|
137
|
+
});
|
|
138
|
+
|
|
139
|
+
// A future reader that reintroduces the dependency on the row fails here
|
|
140
|
+
// rather than silently regressing to write-time resolution (design.md D8).
|
|
141
|
+
test('the resolved cadence follows the manifest, not the row', () => {
|
|
142
|
+
addModule('caddy', '15m');
|
|
143
|
+
addMonitor('caddy', 9999);
|
|
144
|
+
|
|
145
|
+
expect(loadModuleHealthCadences(getDb()).get('caddy')?.cadence).toEqual({ minutes: 15 });
|
|
146
|
+
});
|
|
147
|
+
|
|
148
|
+
test('the resolved cadence follows the override, not the row', () => {
|
|
149
|
+
addModule('caddy', '15m');
|
|
150
|
+
addMonitor('caddy', 9999);
|
|
151
|
+
upsertModuleConfig(getDb(), 'caddy', HEALTH_CHECK_INTERVAL_CONFIG_KEY, '1h');
|
|
152
|
+
|
|
153
|
+
expect(loadModuleHealthCadences(getDb()).get('caddy')?.cadence).toEqual({ minutes: 60 });
|
|
154
|
+
});
|
|
155
|
+
});
|
|
@@ -0,0 +1,90 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Carrying an existing fleet's health-check cadences into `module_configs`.
|
|
3
|
+
*
|
|
4
|
+
* Not bookkeeping. Before this change a `module_hook` monitor's cadence lived
|
|
5
|
+
* on its row, seeded from the manifest at first deploy and editable only by raw
|
|
6
|
+
* SQL; after it, the row is not read and the cadence comes from the operator's
|
|
7
|
+
* override or the manifest's suggestion. So on the FIRST apt upgrade that
|
|
8
|
+
* carries this code, every module whose row diverges from its manifest — which
|
|
9
|
+
* is every module anyone ever re-cadenced by hand — would silently revert to
|
|
10
|
+
* the author's suggestion, and every module an operator deliberately disabled
|
|
11
|
+
* would start being watched again.
|
|
12
|
+
*
|
|
13
|
+
* Two steps, both idempotent, both writing only where no override exists:
|
|
14
|
+
*
|
|
15
|
+
* 1. a row whose cadence differs from what its manifest suggests gets that
|
|
16
|
+
* cadence written as an override;
|
|
17
|
+
* 2. a disabled row gets `manual`.
|
|
18
|
+
*
|
|
19
|
+
* Runs from `celilo system migrate`, which the `.deb` postinst already invokes
|
|
20
|
+
* on every upgrade. Rolling back is safe in both directions: the rows written
|
|
21
|
+
* here are inert to older code, which reads the monitor row, and rolling
|
|
22
|
+
* forward again finds them already present.
|
|
23
|
+
*/
|
|
24
|
+
|
|
25
|
+
import { eq } from 'drizzle-orm';
|
|
26
|
+
import type { DbClient } from '../../db/client';
|
|
27
|
+
import { modules, monitors } from '../../db/schema';
|
|
28
|
+
import type { ModuleManifest } from '../../manifest/schema';
|
|
29
|
+
import { formatCadence, parseCadence } from '../cadence';
|
|
30
|
+
import { getModuleConfigValue, upsertModuleConfig } from '../module-config';
|
|
31
|
+
import { HEALTH_CHECK_INTERVAL_CONFIG_KEY } from './health-cadence';
|
|
32
|
+
|
|
33
|
+
export interface CadenceMigrationReport {
|
|
34
|
+
/** `moduleId → written value`, for the operator-visible summary. */
|
|
35
|
+
written: Map<string, string>;
|
|
36
|
+
/** Modules left alone: an override already existed, or nothing diverged. */
|
|
37
|
+
unchanged: string[];
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
export function migrateMonitorCadences(db: DbClient): CadenceMigrationReport {
|
|
41
|
+
const report: CadenceMigrationReport = { written: new Map(), unchanged: [] };
|
|
42
|
+
|
|
43
|
+
for (const monitor of db.select().from(monitors).where(eq(monitors.kind, 'module_hook')).all()) {
|
|
44
|
+
const moduleId = monitor.target;
|
|
45
|
+
|
|
46
|
+
// An operator who has already set one has said the last word; never
|
|
47
|
+
// overwrite it, which is also what makes a second run a no-op.
|
|
48
|
+
if (getModuleConfigValue(moduleId, HEALTH_CHECK_INTERVAL_CONFIG_KEY, db)) {
|
|
49
|
+
report.unchanged.push(moduleId);
|
|
50
|
+
continue;
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
const module = db.select().from(modules).where(eq(modules.id, moduleId)).get();
|
|
54
|
+
if (!module) {
|
|
55
|
+
// A monitor whose module is gone has nothing to carry over.
|
|
56
|
+
report.unchanged.push(moduleId);
|
|
57
|
+
continue;
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
// Disabled wins over cadence: an operator who stopped watching a module
|
|
61
|
+
// meant that, whatever interval the row happens to carry.
|
|
62
|
+
if (!monitor.enabled) {
|
|
63
|
+
upsertModuleConfig(db, moduleId, HEALTH_CHECK_INTERVAL_CONFIG_KEY, 'manual');
|
|
64
|
+
report.written.set(moduleId, 'manual');
|
|
65
|
+
continue;
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
const manifest = module.manifestData as ModuleManifest;
|
|
69
|
+
const suggested = manifest.hooks?.health_check?.interval;
|
|
70
|
+
const suggestedMinutes = suggested ? parseCadence(suggested) : null;
|
|
71
|
+
const suggestedIsSame =
|
|
72
|
+
suggestedMinutes !== null &&
|
|
73
|
+
suggestedMinutes !== 'manual' &&
|
|
74
|
+
suggestedMinutes.minutes === monitor.intervalMinutes;
|
|
75
|
+
|
|
76
|
+
if (suggestedIsSame) {
|
|
77
|
+
// The row never diverged, so read-time resolution already produces it —
|
|
78
|
+
// and writing an override here would freeze this module against every
|
|
79
|
+
// future manifest correction, which is the failure this change removes.
|
|
80
|
+
report.unchanged.push(moduleId);
|
|
81
|
+
continue;
|
|
82
|
+
}
|
|
83
|
+
|
|
84
|
+
const carried = formatCadence({ minutes: monitor.intervalMinutes });
|
|
85
|
+
upsertModuleConfig(db, moduleId, HEALTH_CHECK_INTERVAL_CONFIG_KEY, carried);
|
|
86
|
+
report.written.set(moduleId, carried);
|
|
87
|
+
}
|
|
88
|
+
|
|
89
|
+
return report;
|
|
90
|
+
}
|