@celilo/cli 0.23.0 → 0.24.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CELILO_CORE_MODULES.md +2 -2
- package/CELILO_SUBSYSTEMS.md +27 -7
- package/package.json +6 -5
- package/src/cli/commands/alerts-act.ts +1 -1
- package/src/cli/commands/backup-create.ts +26 -11
- package/src/cli/commands/backup-list.test.ts +83 -0
- package/src/cli/commands/backup-list.ts +67 -3
- package/src/cli/commands/backup-prune.ts +17 -17
- package/src/cli/commands/backup-sweep.ts +20 -8
- package/src/cli/commands/firewall-interface-list.test.ts +85 -0
- package/src/cli/commands/firewall-interface-list.ts +123 -0
- package/src/cli/commands/machine-add.ts +30 -2
- package/src/cli/commands/module-config.test.ts +70 -3
- package/src/cli/commands/module-config.ts +262 -28
- package/src/cli/commands/module-status.ts +155 -12
- package/src/cli/commands/monitor.ts +116 -19
- package/src/cli/commands/system-migrate.ts +14 -0
- package/src/cli/commands/system-update.ts +4 -1
- package/src/cli/completion.ts +35 -9
- package/src/cli/index.ts +59 -2
- package/src/cli/tui/audit-state.ts +2 -0
- package/src/hooks/capability-loader.ts +130 -4
- package/src/hooks/load-hook-config.test.ts +169 -1
- package/src/hooks/load-hook-config.ts +118 -20
- package/src/hooks/types.ts +2 -1
- package/src/manifest/contracts/v1.ts +16 -0
- package/src/manifest/schema.ts +40 -65
- package/src/services/alerting/builtin-monitors.test.ts +18 -10
- package/src/services/alerting/cadence-migration.test.ts +155 -0
- package/src/services/alerting/cadence-migration.ts +90 -0
- package/src/services/alerting/coverage-source.ts +8 -11
- package/src/services/alerting/deploy-hooks.test.ts +16 -7
- package/src/services/alerting/deploy-hooks.ts +11 -5
- package/src/services/alerting/health-cadence.test.ts +58 -0
- package/src/services/alerting/health-cadence.ts +128 -0
- package/src/services/alerting/health-coverage.ts +18 -8
- package/src/services/alerting/monitors.ts +50 -15
- package/src/services/alerting/sweep-runner.test.ts +51 -3
- package/src/services/alerting/sweep-runner.ts +30 -7
- package/src/services/audit/backup-source.ts +24 -1
- package/src/services/audit/backups.test.ts +95 -10
- package/src/services/audit/backups.ts +40 -37
- package/src/services/audit/interface-classification.test.ts +220 -0
- package/src/services/audit/interface-classification.ts +167 -0
- package/src/services/audit/types.ts +2 -1
- package/src/services/backup-age-agreement.test.ts +118 -0
- package/src/services/backup-create.ts +36 -30
- package/src/services/backup-metadata.ts +52 -1
- package/src/services/backup-retention.test.ts +123 -0
- package/src/services/backup-retention.ts +66 -5
- package/src/services/backup-schedule.test.ts +166 -0
- package/src/services/backup-schedule.ts +105 -15
- package/src/services/backup-staging.ts +14 -1
- package/src/services/backup-sweep.test.ts +22 -3
- package/src/services/backup-sweep.ts +15 -5
- package/src/services/cadence.test.ts +97 -0
- package/src/services/cadence.ts +165 -0
- package/src/services/config-provenance.test.ts +155 -0
- package/src/services/config-provenance.ts +104 -0
- package/src/services/machine-detector.ts +23 -1
- package/src/services/module-config.ts +33 -0
- package/src/services/storage-providers/s3.test.ts +96 -13
- package/src/services/storage-providers/s3.ts +48 -15
- package/src/services/zone-detector.test.ts +34 -3
- package/src/services/zone-detector.ts +33 -13
- package/src/variables/context.ts +69 -15
- package/src/variables/declarative-derivation.test.ts +53 -0
- package/src/variables/declarative-derivation.ts +13 -2
package/src/cli/index.ts
CHANGED
|
@@ -48,6 +48,7 @@ import {
|
|
|
48
48
|
handleEventsTail,
|
|
49
49
|
handleEventsUninstallDaemon,
|
|
50
50
|
} from './commands/events';
|
|
51
|
+
import { handleFirewallInterfaceList } from './commands/firewall-interface-list';
|
|
51
52
|
import { handleHookRun } from './commands/hook-run';
|
|
52
53
|
import {
|
|
53
54
|
handleIpamIpListReservations,
|
|
@@ -68,7 +69,11 @@ import { moduleAudit } from './commands/module-audit';
|
|
|
68
69
|
import { handleModuleBuild } from './commands/module-build';
|
|
69
70
|
import { handleModuleChangeset } from './commands/module-changeset';
|
|
70
71
|
import { handleModuleCheck } from './commands/module-check';
|
|
71
|
-
import {
|
|
72
|
+
import {
|
|
73
|
+
handleModuleConfigGet,
|
|
74
|
+
handleModuleConfigSet,
|
|
75
|
+
handleModuleConfigUnset,
|
|
76
|
+
} from './commands/module-config';
|
|
72
77
|
import { handleModuleDeploy } from './commands/module-deploy';
|
|
73
78
|
import { handleModuleGenerate } from './commands/module-generate';
|
|
74
79
|
import { handleModuleHealth } from './commands/module-health';
|
|
@@ -204,6 +209,7 @@ Commands:
|
|
|
204
209
|
storage Manage backup storage destinations
|
|
205
210
|
backup Create and manage backups
|
|
206
211
|
restore Restore a celilo-mgmt backup from a local file (fresh-bootstrap path)
|
|
212
|
+
firewall Inspect a firewall's interfaces (classification, on demand)
|
|
207
213
|
machine Manage machine pool (bring-your-own-hardware)
|
|
208
214
|
system Manage system configuration
|
|
209
215
|
apt-upgrade Upgrade the deb-installed celilo packages + apply migrations
|
|
@@ -598,6 +604,7 @@ Subcommands:
|
|
|
598
604
|
|
|
599
605
|
config set <id> <key> <value> Set module configuration value
|
|
600
606
|
config get <id> [key] Get module configuration value(s)
|
|
607
|
+
config unset <id> <key> Remove an override; follow the manifest again
|
|
601
608
|
|
|
602
609
|
secret set <id> <key> <value> Set encrypted module secret
|
|
603
610
|
secret list <id> List module secrets
|
|
@@ -675,6 +682,8 @@ Examples:
|
|
|
675
682
|
celilo module config set homebridge container_ip "192.168.0.110/24"
|
|
676
683
|
celilo module config get homebridge
|
|
677
684
|
celilo module config get homebridge hostname
|
|
685
|
+
celilo module config set caddy backup_schedule 6h
|
|
686
|
+
celilo module config unset caddy backup_schedule
|
|
678
687
|
celilo module show-config homebridge
|
|
679
688
|
celilo module show-zone homebridge
|
|
680
689
|
celilo module build caddy
|
|
@@ -917,6 +926,28 @@ Related Commands:
|
|
|
917
926
|
/**
|
|
918
927
|
* Display machine command help
|
|
919
928
|
*/
|
|
929
|
+
function displayFirewallHelp(): CommandResult {
|
|
930
|
+
console.log(`
|
|
931
|
+
celilo firewall — inspect a firewall's interfaces
|
|
932
|
+
|
|
933
|
+
Usage:
|
|
934
|
+
celilo firewall interface list [<hostname>]
|
|
935
|
+
|
|
936
|
+
Shows how celilo classifies every interface on a firewall: the zone it
|
|
937
|
+
matched, the external edge, or ALIEN when nothing accounts for it.
|
|
938
|
+
|
|
939
|
+
Read-only. It reads the recorded interface table and the declared zone
|
|
940
|
+
subnets and classifies in memory — it never touches the box, so it is safe
|
|
941
|
+
to run against a firewall whose converge is currently refusing.
|
|
942
|
+
|
|
943
|
+
With no hostname, every machine celilo classifies as a router is shown.
|
|
944
|
+
|
|
945
|
+
An interface celilo cannot attribute is resolved by DECLARING it:
|
|
946
|
+
celilo system config set network.<zone>.subnet <cidr>
|
|
947
|
+
`);
|
|
948
|
+
return { success: true, message: 'firewall help' };
|
|
949
|
+
}
|
|
950
|
+
|
|
920
951
|
function displayMachineHelp(): CommandResult {
|
|
921
952
|
const helpText = `
|
|
922
953
|
Celilo - Machine Pool Management
|
|
@@ -1544,7 +1575,7 @@ export async function runCli(argv: string[]): Promise<CommandResult> {
|
|
|
1544
1575
|
if (!configSubcommand) {
|
|
1545
1576
|
return {
|
|
1546
1577
|
success: false,
|
|
1547
|
-
error: 'Config action required (set or
|
|
1578
|
+
error: 'Config action required (set, get or unset)\n\nRun "celilo help" for usage',
|
|
1548
1579
|
};
|
|
1549
1580
|
}
|
|
1550
1581
|
const configArgs = parsed.args.slice(1);
|
|
@@ -1554,6 +1585,9 @@ export async function runCli(argv: string[]): Promise<CommandResult> {
|
|
|
1554
1585
|
if (configSubcommand === 'get') {
|
|
1555
1586
|
return handleModuleConfigGet(configArgs);
|
|
1556
1587
|
}
|
|
1588
|
+
if (configSubcommand === 'unset') {
|
|
1589
|
+
return handleModuleConfigUnset(configArgs);
|
|
1590
|
+
}
|
|
1557
1591
|
return {
|
|
1558
1592
|
success: false,
|
|
1559
1593
|
error: `Unknown config action: ${configSubcommand}`,
|
|
@@ -2050,6 +2084,29 @@ export async function runCli(argv: string[]): Promise<CommandResult> {
|
|
|
2050
2084
|
};
|
|
2051
2085
|
}
|
|
2052
2086
|
|
|
2087
|
+
if (parsed.command === 'firewall') {
|
|
2088
|
+
if (parsed.flags.help || parsed.flags.h || !parsed.subcommand) {
|
|
2089
|
+
return displayFirewallHelp();
|
|
2090
|
+
}
|
|
2091
|
+
if (parsed.subcommand === 'interface') {
|
|
2092
|
+
// `interface` is a group, not a leaf: `list` is its only verb today.
|
|
2093
|
+
// `acknowledge` and `enforce` were both deleted in the design amendments —
|
|
2094
|
+
// there is no policy toggle and no per-interface escape hatch.
|
|
2095
|
+
const verb = parsed.args[0];
|
|
2096
|
+
if (!verb || verb === 'list') {
|
|
2097
|
+
return handleFirewallInterfaceList(parsed.args.slice(1), parsed.flags);
|
|
2098
|
+
}
|
|
2099
|
+
return {
|
|
2100
|
+
success: false,
|
|
2101
|
+
error: `Unknown firewall interface subcommand: ${verb}\n\nRun "celilo firewall --help" for usage`,
|
|
2102
|
+
};
|
|
2103
|
+
}
|
|
2104
|
+
return {
|
|
2105
|
+
success: false,
|
|
2106
|
+
error: `Unknown firewall subcommand: ${parsed.subcommand}\n\nRun "celilo firewall --help" for usage`,
|
|
2107
|
+
};
|
|
2108
|
+
}
|
|
2109
|
+
|
|
2053
2110
|
if (parsed.command === 'machine') {
|
|
2054
2111
|
// Handle machine --help
|
|
2055
2112
|
if (parsed.flags.help || parsed.flags.h) {
|
|
@@ -88,9 +88,11 @@ export const ALL_CATEGORIES: readonly DriftCategory[] = [
|
|
|
88
88
|
'disk_space',
|
|
89
89
|
'transport_reads',
|
|
90
90
|
'trusted_sources',
|
|
91
|
+
'interface_classification',
|
|
91
92
|
];
|
|
92
93
|
|
|
93
94
|
export const CATEGORY_LABELS: Record<DriftCategory, string> = {
|
|
95
|
+
interface_classification: 'Firewall interfaces',
|
|
94
96
|
cli_version: 'CLI version',
|
|
95
97
|
schema: 'Schema migrations',
|
|
96
98
|
capability_abi: 'Capability ABI',
|
|
@@ -154,6 +154,32 @@ export async function resolveFirewallNatIp(db: DbClient): Promise<string | undef
|
|
|
154
154
|
return undefined;
|
|
155
155
|
}
|
|
156
156
|
|
|
157
|
+
/**
|
|
158
|
+
* The addresses of every machine a `firewall` capability provider manages.
|
|
159
|
+
*
|
|
160
|
+
* This is the authoritative answer to "which machines are firewalls", and it has
|
|
161
|
+
* to be: a machine's stored `role` is decided by `machine add`, from the zones
|
|
162
|
+
* declared AT THAT MOMENT. The normal order is `machine add` and THEN deploy
|
|
163
|
+
* iptables, whose `on_install` writes the zone subnets — so the firewall is
|
|
164
|
+
* recorded as a plain host and stays that way. A firewall provider naming an IP
|
|
165
|
+
* is a statement, not an inference from a snapshot.
|
|
166
|
+
*/
|
|
167
|
+
export async function listFirewallIps(db: DbClient): Promise<string[]> {
|
|
168
|
+
const providers = db
|
|
169
|
+
.select()
|
|
170
|
+
.from(capabilities)
|
|
171
|
+
.where(eq(capabilities.capabilityName, 'firewall'))
|
|
172
|
+
.all();
|
|
173
|
+
const ips: string[] = [];
|
|
174
|
+
for (const provider of providers) {
|
|
175
|
+
const config = await loadModuleConfig(provider.moduleId, db);
|
|
176
|
+
if (typeof config.firewall_ip === 'string' && config.firewall_ip) {
|
|
177
|
+
ips.push(config.firewall_ip);
|
|
178
|
+
}
|
|
179
|
+
}
|
|
180
|
+
return ips;
|
|
181
|
+
}
|
|
182
|
+
|
|
157
183
|
/**
|
|
158
184
|
* Caddy's zone-routable IP — its own DMZ ingress address (`target_ip`, the same
|
|
159
185
|
* value public_web exposes as `dmz_ip`). This is the in-zone split-horizon
|
|
@@ -606,6 +632,9 @@ function buildCapabilityInterface(
|
|
|
606
632
|
undefined, // no upstream — the chain path handles that
|
|
607
633
|
undefined, // logger is applied by wrapWithLogging at the loader site
|
|
608
634
|
trustedSourceStore,
|
|
635
|
+
// LIVE, for the same reason as the chain path: a declaration written
|
|
636
|
+
// during this hook run must be visible to the converge that follows it.
|
|
637
|
+
zones ? { list: zones.declaredNetworks } : undefined,
|
|
609
638
|
);
|
|
610
639
|
return stampProvider(iface, providerModuleId);
|
|
611
640
|
}
|
|
@@ -707,6 +736,18 @@ interface FirewallZones {
|
|
|
707
736
|
trustedSubnets: string[];
|
|
708
737
|
/** celilo's control-plane network, as a DESTINATION for trusted sources. */
|
|
709
738
|
controlPlaneSubnet?: string;
|
|
739
|
+
/**
|
|
740
|
+
* Every zone with a declared subnet, WITH its name — the input to interface
|
|
741
|
+
* classification. Excludes `external`, which is a residual rather than a
|
|
742
|
+
* subnet (design D1, amended).
|
|
743
|
+
*/
|
|
744
|
+
/**
|
|
745
|
+
* A LIVE read, not a snapshot. Everything else here is captured when the
|
|
746
|
+
* capability is built — before any hook runs — and for declarations that is
|
|
747
|
+
* wrong: `wireguard` declares its VPN subnet and brings `wg0` up inside one
|
|
748
|
+
* hook run, and the converge that follows has to see the declaration.
|
|
749
|
+
*/
|
|
750
|
+
declaredNetworks: () => Array<{ zone: string; subnet: string }>;
|
|
710
751
|
/**
|
|
711
752
|
* Every declared zone subnet — read from
|
|
712
753
|
* `network.<zone>.subnet` across all of `NETWORK_ZONES`, not just the three
|
|
@@ -806,6 +847,35 @@ export function loadTrustedSubnets(db: DbClient, firewallIp?: string): TrustedSu
|
|
|
806
847
|
* of them. Zones with no configured subnet are omitted — their traffic stays
|
|
807
848
|
* denied (fail-closed).
|
|
808
849
|
*/
|
|
850
|
+
/**
|
|
851
|
+
* Every network celilo can attribute an interface to: one entry per
|
|
852
|
+
* `network.<name>.subnet` in system config.
|
|
853
|
+
*
|
|
854
|
+
* Read from the config rather than from `NETWORK_ZONES`, because celilo declares
|
|
855
|
+
* networks that are not placement zones. `wireguard`'s `on_install` writes
|
|
856
|
+
* `network.control-plane-vpn.subnet` before it brings `wg0` up, exactly as the
|
|
857
|
+
* design requires (D3), and the firewall is supposed to classify `wg0` "by the
|
|
858
|
+
* same subnet containment it uses for every other leg". Reading only
|
|
859
|
+
* `NETWORK_ZONES` left that declaration invisible: the converge could not
|
|
860
|
+
* attribute `wg0` and refused — and on a firewall with a baseline it would have
|
|
861
|
+
* ISOLATED it, shutting down the admin VPN.
|
|
862
|
+
*
|
|
863
|
+
* `external` is deliberately absent even when something has set
|
|
864
|
+
* `network.external.subnet`: it is the RESIDUAL, decided by `isPubliclyRoutable`
|
|
865
|
+
* and never by containment (design D1, amended). Matching an interface to
|
|
866
|
+
* `external` by subnet would reintroduce the overload this change removes.
|
|
867
|
+
*/
|
|
868
|
+
export function readDeclaredNetworks(db: DbClient): Array<{ zone: string; subnet: string }> {
|
|
869
|
+
return db
|
|
870
|
+
.select()
|
|
871
|
+
.from(systemConfig)
|
|
872
|
+
.all()
|
|
873
|
+
.flatMap((row) => {
|
|
874
|
+
const zone = /^network\.(.+)\.subnet$/.exec(row.key)?.[1];
|
|
875
|
+
return zone && zone !== 'external' && row.value ? [{ zone, subnet: row.value }] : [];
|
|
876
|
+
});
|
|
877
|
+
}
|
|
878
|
+
|
|
809
879
|
function loadFirewallZones(db: DbClient): FirewallZones {
|
|
810
880
|
const zoneTiers: Array<{ name: string; subnet: string }> = [];
|
|
811
881
|
for (const zone of ZONE_TIER_ORDER) {
|
|
@@ -822,17 +892,65 @@ function loadFirewallZones(db: DbClient): FirewallZones {
|
|
|
822
892
|
// snapshot put the subnet straight back, so the set could grow but never
|
|
823
893
|
// shrink. That is exactly what happened: `on_uninstall` withdrew the VPN's
|
|
824
894
|
// trusted source, logged success, and the reach rules were re-rendered anyway.
|
|
895
|
+
// Every zone that HAS a declared subnet, keeping the zone NAME. Interface
|
|
896
|
+
// classification needs the name — `external` and `internal` are the only two
|
|
897
|
+
// legs a default route may leave through (design D11), which is not a question
|
|
898
|
+
// a bare list of subnets can answer.
|
|
899
|
+
//
|
|
900
|
+
// `external` is deliberately absent from this list even when something has set
|
|
901
|
+
// `network.external.subnet`: `external` is the RESIDUAL, decided by
|
|
902
|
+
// `isPubliclyRoutable`, never by containment (design D1, amended). Matching an
|
|
903
|
+
// interface to `external` by subnet would reintroduce the overload this change
|
|
904
|
+
// removes.
|
|
905
|
+
//
|
|
906
|
+
// EVERY declared network, not just the six `NETWORK_ZONES`. celilo declares
|
|
907
|
+
// networks that are not placement zones — `network.control-plane-vpn.subnet`,
|
|
908
|
+
// written by `wireguard`'s `on_install` — and the design is explicit that the
|
|
909
|
+
// firewall "classifies `wg0` by the same subnet containment it uses for every
|
|
910
|
+
// other leg". Reading only `NETWORK_ZONES` left that declaration invisible:
|
|
911
|
+
// the module declared the subnet before bringing the interface up, exactly as
|
|
912
|
+
// designed, and the converge still could not attribute `wg0` and refused. On a
|
|
913
|
+
// firewall that already had a baseline it would have gone further and ISOLATED
|
|
914
|
+
// the interface — celilo shutting down the admin VPN, which for a remote
|
|
915
|
+
// operator is the way back in.
|
|
916
|
+
//
|
|
917
|
+
// So the source is the config itself. A hardcoded list here could only ever
|
|
918
|
+
// describe the networks celilo knew about when this line was written.
|
|
919
|
+
|
|
825
920
|
return {
|
|
826
921
|
zoneTiers,
|
|
827
922
|
trustedSubnets: loadTrustedSubnets(db).map((e) => e.subnet),
|
|
828
923
|
controlPlaneSubnet: loadControlPlaneSubnet(db) ?? readZoneSubnet(db, 'internal'),
|
|
829
|
-
|
|
830
|
-
|
|
831
|
-
|
|
832
|
-
|
|
924
|
+
declaredNetworks: () => readDeclaredNetworks(db),
|
|
925
|
+
// Placement ZONES only — deliberately NOT `declaredNetworks`, which is wider.
|
|
926
|
+
// Fronting a subnet renders translation for it; the control-plane VPN
|
|
927
|
+
// reaches the fleet as a trusted source instead, and giving it a second
|
|
928
|
+
// mechanism would change the rendered ruleset for every existing fleet.
|
|
929
|
+
frontedSubnets: NETWORK_ZONES.filter((zone) => zone !== 'external').flatMap((zone) => {
|
|
930
|
+
const subnet = readZoneSubnet(db, zone);
|
|
931
|
+
return subnet ? [subnet] : [];
|
|
932
|
+
}),
|
|
833
933
|
};
|
|
834
934
|
}
|
|
835
935
|
|
|
936
|
+
/**
|
|
937
|
+
* The interface baseline as stored in module config: a comma-separated list of
|
|
938
|
+
* interface names, or absent.
|
|
939
|
+
*
|
|
940
|
+
* Absent and empty are the SAME thing here and both mean "no baseline" — a
|
|
941
|
+
* firewall with zero interfaces does not exist, so an empty string can only be
|
|
942
|
+
* a cleared or never-written value. Treating it as a baseline of nothing would
|
|
943
|
+
* mean every interface is "new", which is the opposite of what it says.
|
|
944
|
+
*/
|
|
945
|
+
function parseInterfaceBaseline(raw: unknown): string[] | undefined {
|
|
946
|
+
if (typeof raw !== 'string') return undefined;
|
|
947
|
+
const names = raw
|
|
948
|
+
.split(',')
|
|
949
|
+
.map((n) => n.trim())
|
|
950
|
+
.filter((n) => n.length > 0);
|
|
951
|
+
return names.length > 0 ? names : undefined;
|
|
952
|
+
}
|
|
953
|
+
|
|
836
954
|
async function buildFirewallChain(
|
|
837
955
|
allProviders: Array<{
|
|
838
956
|
id: number;
|
|
@@ -982,11 +1100,19 @@ async function buildFirewallChain(
|
|
|
982
1100
|
trustedSubnets: zones.trustedSubnets,
|
|
983
1101
|
controlPlaneSubnet: zones.controlPlaneSubnet,
|
|
984
1102
|
frontedSubnets: zones.frontedSubnets,
|
|
1103
|
+
// The recorded baseline (D12), from this firewall's own module config.
|
|
1104
|
+
// Absent means the box has never converged cleanly, so an interface
|
|
1105
|
+
// celilo cannot attribute refuses rather than being disabled.
|
|
1106
|
+
interfaceBaseline: parseInterfaceBaseline(provConfig.interface_baseline),
|
|
985
1107
|
},
|
|
986
1108
|
store,
|
|
987
1109
|
currentUpstream,
|
|
988
1110
|
logger,
|
|
989
1111
|
trustedSourceStore,
|
|
1112
|
+
// LIVE, not a snapshot: a declaration written during this hook run —
|
|
1113
|
+
// `wireguard` declaring its VPN subnet before creating `wg0` — must be
|
|
1114
|
+
// visible to the converge that follows it in the same run.
|
|
1115
|
+
{ list: zones.declaredNetworks },
|
|
990
1116
|
);
|
|
991
1117
|
|
|
992
1118
|
debugLog(`firewall chain: wired ${provider.moduleId} → ${hasExternal.moduleId}`);
|
|
@@ -17,8 +17,9 @@
|
|
|
17
17
|
*/
|
|
18
18
|
|
|
19
19
|
import { afterEach, beforeEach, describe, expect, test } from 'bun:test';
|
|
20
|
+
import { eq } from 'drizzle-orm';
|
|
20
21
|
import type { DbClient } from '../db/client';
|
|
21
|
-
import { machines, moduleInfrastructure, modules } from '../db/schema';
|
|
22
|
+
import { machines, moduleConfigs, moduleInfrastructure, modules, systemConfig } from '../db/schema';
|
|
22
23
|
import { upsertModuleConfig } from '../services/module-config';
|
|
23
24
|
import { cleanupTestDatabase, setupTestDatabase } from '../test-utils/database';
|
|
24
25
|
import { loadHookConfigMap } from './load-hook-config';
|
|
@@ -165,3 +166,170 @@ describe('loadHookConfigMap', () => {
|
|
|
165
166
|
expect(result['ip.primary']).toBeUndefined();
|
|
166
167
|
});
|
|
167
168
|
});
|
|
169
|
+
|
|
170
|
+
/**
|
|
171
|
+
* The 2026-08-14 outage, as a test.
|
|
172
|
+
*
|
|
173
|
+
* `technitium` declares `vpn_subnet` with `source: system` and
|
|
174
|
+
* `derive_from: $system:network.control-plane-vpn.subnet`. The system key was
|
|
175
|
+
* set on the fleet AFTER the module had already been configured, so no
|
|
176
|
+
* `module_configs` row was ever written for it — `applyDeclarativeDerivations`
|
|
177
|
+
* skips a `source: system` variable once the module has any stored config for
|
|
178
|
+
* it, and nothing re-derives one that is absent. The variable is
|
|
179
|
+
* `required: false`, so deploy-validation passed without comment.
|
|
180
|
+
*
|
|
181
|
+
* The consumer was a capability factory, which gets its config from
|
|
182
|
+
* `loadHookConfigMap`. A direct `module_configs` select cannot see a value that
|
|
183
|
+
* was never stored, so the factory built split-horizon DNS with no view for the
|
|
184
|
+
* admin VPN: queries from it matched nothing, returned NOERROR with zero
|
|
185
|
+
* records, fell through to public DNS, and the operator could not reach the
|
|
186
|
+
* forge while every service reported healthy.
|
|
187
|
+
*
|
|
188
|
+
* The fix is that the hook config map is built from the resolution context —
|
|
189
|
+
* which recomputes derived values on every build — not from the table alone.
|
|
190
|
+
*/
|
|
191
|
+
describe('loadHookConfigMap: derived values the config table never stored', () => {
|
|
192
|
+
let db: DbClient;
|
|
193
|
+
|
|
194
|
+
const VPN_SUBNET_KEY = 'network.control-plane-vpn.subnet';
|
|
195
|
+
const VPN_SUBNET = '10.255.255.0/24';
|
|
196
|
+
|
|
197
|
+
beforeEach(async () => {
|
|
198
|
+
db = await setupTestDatabase();
|
|
199
|
+
db.insert(modules)
|
|
200
|
+
.values({
|
|
201
|
+
id: 'technitium',
|
|
202
|
+
name: 'Technitium DNS',
|
|
203
|
+
version: '1.0.0',
|
|
204
|
+
sourcePath: '/tmp/technitium',
|
|
205
|
+
manifestData: {
|
|
206
|
+
variables: {
|
|
207
|
+
owns: [
|
|
208
|
+
{ name: 'hostname', type: 'string', source: 'user' },
|
|
209
|
+
{
|
|
210
|
+
name: 'vpn_subnet',
|
|
211
|
+
type: 'string',
|
|
212
|
+
source: 'system',
|
|
213
|
+
required: false,
|
|
214
|
+
derive_from: `$system:${VPN_SUBNET_KEY}`,
|
|
215
|
+
},
|
|
216
|
+
],
|
|
217
|
+
},
|
|
218
|
+
},
|
|
219
|
+
})
|
|
220
|
+
.run();
|
|
221
|
+
// The module was configured first: it has its user rows, and no row for
|
|
222
|
+
// the derived variable.
|
|
223
|
+
upsertModuleConfig(db, 'technitium', 'hostname', 'dns-int');
|
|
224
|
+
});
|
|
225
|
+
|
|
226
|
+
afterEach(async () => {
|
|
227
|
+
await cleanupTestDatabase(db);
|
|
228
|
+
});
|
|
229
|
+
|
|
230
|
+
test('a hook sees a source:system derive whose system key was set after the module was configured', async () => {
|
|
231
|
+
db.insert(systemConfig).values({ key: VPN_SUBNET_KEY, value: VPN_SUBNET }).run();
|
|
232
|
+
|
|
233
|
+
const result = await loadHookConfigMap('technitium', db);
|
|
234
|
+
|
|
235
|
+
expect(result.hostname).toBe('dns-int');
|
|
236
|
+
expect(result.vpn_subnet).toBe(VPN_SUBNET);
|
|
237
|
+
});
|
|
238
|
+
|
|
239
|
+
test('recomputing does not write the derived value back into module_configs', async () => {
|
|
240
|
+
db.insert(systemConfig).values({ key: VPN_SUBNET_KEY, value: VPN_SUBNET }).run();
|
|
241
|
+
|
|
242
|
+
await loadHookConfigMap('technitium', db);
|
|
243
|
+
|
|
244
|
+
// Reading a hook's config is not a deploy. It must not seed or refresh
|
|
245
|
+
// stored config as a side effect — that is how a stale snapshot gets
|
|
246
|
+
// written in the first place.
|
|
247
|
+
const stored = db
|
|
248
|
+
.select()
|
|
249
|
+
.from(moduleConfigs)
|
|
250
|
+
.where(eq(moduleConfigs.moduleId, 'technitium'))
|
|
251
|
+
.all();
|
|
252
|
+
expect(stored.map((row) => row.key).sort()).toEqual(['hostname']);
|
|
253
|
+
});
|
|
254
|
+
|
|
255
|
+
test('an optional derive whose system key is still unset stays absent, not an error', async () => {
|
|
256
|
+
// No systemConfig row at all — the state the fleet was in before the VPN
|
|
257
|
+
// subnet was declared. `required: false`, so this is silence, not failure.
|
|
258
|
+
const result = await loadHookConfigMap('technitium', db);
|
|
259
|
+
|
|
260
|
+
expect(result.hostname).toBe('dns-int');
|
|
261
|
+
expect(result.vpn_subnet).toBeUndefined();
|
|
262
|
+
});
|
|
263
|
+
|
|
264
|
+
test('an already-stored value reaches the hook unchanged', async () => {
|
|
265
|
+
// Precedence: a stored row wins, and the recomputed context only fills
|
|
266
|
+
// keys the table has no row for. Every derived value on the fleet today is
|
|
267
|
+
// stored, so this is what keeps the new read path from changing what any
|
|
268
|
+
// deployed hook already receives — the change is purely additive here.
|
|
269
|
+
// (Once derived values stop being persisted, the stored row disappears and
|
|
270
|
+
// the recomputed value is all that is left.)
|
|
271
|
+
db.insert(systemConfig).values({ key: VPN_SUBNET_KEY, value: VPN_SUBNET }).run();
|
|
272
|
+
upsertModuleConfig(db, 'technitium', 'vpn_subnet', '10.99.0.0/24');
|
|
273
|
+
|
|
274
|
+
const result = await loadHookConfigMap('technitium', db);
|
|
275
|
+
|
|
276
|
+
expect(result.vpn_subnet).toBe('10.99.0.0/24');
|
|
277
|
+
});
|
|
278
|
+
});
|
|
279
|
+
|
|
280
|
+
/**
|
|
281
|
+
* Recomputation must not make a hook's config LESS available than reading the
|
|
282
|
+
* table did.
|
|
283
|
+
*
|
|
284
|
+
* `applyDeclarativeDerivations` throws when a `required: true` variable's
|
|
285
|
+
* derivation fails, which at generate time is exactly right — a deploy that
|
|
286
|
+
* cannot resolve a required value should stop. But this reader also serves
|
|
287
|
+
* health checks and `module run-hook`, which used to be unable to fail this
|
|
288
|
+
* way at all: they read stored rows, and a stored row cannot throw. A provider
|
|
289
|
+
* that is paused, removed, or not yet deployed would take its consumers'
|
|
290
|
+
* health checks down with it — reporting the consumer as broken when the
|
|
291
|
+
* consumer is fine.
|
|
292
|
+
*/
|
|
293
|
+
describe('loadHookConfigMap: a failing derive does not take the hook down', () => {
|
|
294
|
+
let db: DbClient;
|
|
295
|
+
|
|
296
|
+
beforeEach(async () => {
|
|
297
|
+
db = await setupTestDatabase();
|
|
298
|
+
db.insert(modules)
|
|
299
|
+
.values({
|
|
300
|
+
id: 'caddy',
|
|
301
|
+
name: 'Caddy',
|
|
302
|
+
version: '1.0.0',
|
|
303
|
+
sourcePath: '/tmp/caddy',
|
|
304
|
+
manifestData: {
|
|
305
|
+
variables: {
|
|
306
|
+
owns: [
|
|
307
|
+
{ name: 'hostname', type: 'string', source: 'user' },
|
|
308
|
+
{
|
|
309
|
+
name: 'primary_domain',
|
|
310
|
+
type: 'string',
|
|
311
|
+
source: 'capability',
|
|
312
|
+
required: true,
|
|
313
|
+
derive_from: '$capability:dns_registrar.zone.primary_domain',
|
|
314
|
+
},
|
|
315
|
+
],
|
|
316
|
+
},
|
|
317
|
+
},
|
|
318
|
+
})
|
|
319
|
+
.run();
|
|
320
|
+
upsertModuleConfig(db, 'caddy', 'hostname', 'caddy');
|
|
321
|
+
});
|
|
322
|
+
|
|
323
|
+
afterEach(async () => {
|
|
324
|
+
await cleanupTestDatabase(db);
|
|
325
|
+
});
|
|
326
|
+
|
|
327
|
+
test('stored config still reaches the hook when a required derive cannot resolve', async () => {
|
|
328
|
+
// No `dns_registrar` capability is registered — the provider is paused,
|
|
329
|
+
// removed, or has not been deployed yet.
|
|
330
|
+
const result = await loadHookConfigMap('caddy', db);
|
|
331
|
+
|
|
332
|
+
expect(result.hostname).toBe('caddy');
|
|
333
|
+
expect(result.primary_domain).toBeUndefined();
|
|
334
|
+
});
|
|
335
|
+
});
|
|
@@ -11,30 +11,65 @@
|
|
|
11
11
|
* `capability-loader.ts:loadModuleConfig` had — same shape, different
|
|
12
12
|
* code, easy to miss.
|
|
13
13
|
*
|
|
14
|
-
* The shape:
|
|
15
|
-
*
|
|
16
|
-
*
|
|
17
|
-
*
|
|
18
|
-
*
|
|
19
|
-
*
|
|
20
|
-
*
|
|
21
|
-
*
|
|
22
|
-
*
|
|
23
|
-
*
|
|
24
|
-
*
|
|
25
|
-
*
|
|
26
|
-
*
|
|
27
|
-
*
|
|
28
|
-
*
|
|
14
|
+
* The shape, in the order the layers are applied:
|
|
15
|
+
* 1. Every row from `module_configs`, parsed from `valueJson` via
|
|
16
|
+
* the shared `parseStoredConfigValue` helper. This preserves the
|
|
17
|
+
* types declared in each module's manifest: `number` reads as
|
|
18
|
+
* `number`, `boolean` as `boolean`, complex types as their
|
|
19
|
+
* parsed JSON shape. Pre-Defect-1, this path returned raw strings
|
|
20
|
+
* for primitives (because `valueJson` was NULL for them); that
|
|
21
|
+
* broke capability calls like `firewall.exposeService({ports:[...]})`
|
|
22
|
+
* that did a `typeof === 'number'` check downstream. Fixed in
|
|
23
|
+
* v2 by always populating valueJson on write.
|
|
24
|
+
* 2. If `target_ip` isn't in the row set, look up the deployment
|
|
25
|
+
* machine and inject both `target_ip` AND `ip.primary` from
|
|
26
|
+
* `machines.ipAddress`. Two keys because consumers historically used
|
|
27
|
+
* either name (e.g. caddy's `setup-network.ts` checks
|
|
28
|
+
* `target_ip || ip.primary`); fixing the drift means filling both.
|
|
29
|
+
* 3. For each variable the module's manifest declares, the value the
|
|
30
|
+
* resolution context currently computes — but only where the first two
|
|
31
|
+
* layers left that key unset. See below.
|
|
29
32
|
*
|
|
30
33
|
* Container deploys write `target_ip` into `module_configs` explicitly
|
|
31
34
|
* during generate/deploy, so the fallback only fires for machine
|
|
32
35
|
* deploys (existing iron, no terraform — what every e2e test uses).
|
|
36
|
+
*
|
|
37
|
+
* ## Why layer 3 exists
|
|
38
|
+
*
|
|
39
|
+
* A derived variable — one whose value comes from system config, a
|
|
40
|
+
* capability, or the selected infrastructure rather than from the operator —
|
|
41
|
+
* only reaches a hook through this map. Reading `module_configs` alone can
|
|
42
|
+
* only see the derives that happen to have been WRITTEN there, and one that
|
|
43
|
+
* was never written is indistinguishable from one that does not exist.
|
|
44
|
+
*
|
|
45
|
+
* On 2026-08-14 that was not a theoretical gap. `technitium`'s `vpn_subnet`
|
|
46
|
+
* (`source: system`) had no row, because the system key was set after the
|
|
47
|
+
* module was configured and nothing re-derives a `source: system` value that
|
|
48
|
+
* is already absent. Being `required: false`, deploy-validation passed in
|
|
49
|
+
* silence. The capability factory that builds split-horizon DNS therefore
|
|
50
|
+
* built no view for the admin VPN; queries from it matched nothing, returned
|
|
51
|
+
* NOERROR with zero records, fell through to public DNS, and the operator
|
|
52
|
+
* could not reach the forge while every service reported healthy.
|
|
53
|
+
*
|
|
54
|
+
* So the map is completed from `readResolutionContext` — the same computation
|
|
55
|
+
* a build does, run without any of a build's side effects. Stored rows still
|
|
56
|
+
* win, which is what makes this purely additive for every value already on
|
|
57
|
+
* the fleet; the context only supplies what the table is missing. Context
|
|
58
|
+
* values arrive as strings (declarative derivation resolves string templates
|
|
59
|
+
* only), so this layer never overwrites a typed row with a stringified one.
|
|
60
|
+
*
|
|
61
|
+
* It is narrowed to the manifest's declared variables on purpose. The
|
|
62
|
+
* resolution context also carries values that exist to drive template
|
|
63
|
+
* generation — `inventory.*`, `requires.system.*`, `lxc_nameserver` — which a
|
|
64
|
+
* hook has no business reading and which have never appeared in this map.
|
|
65
|
+
* Widening a hook's view of the world is not what this layer is for.
|
|
33
66
|
*/
|
|
34
67
|
import { eq } from 'drizzle-orm';
|
|
35
68
|
import type { DbClient } from '../db/client';
|
|
36
|
-
import { machines, moduleConfigs, moduleInfrastructure } from '../db/schema';
|
|
69
|
+
import { machines, moduleConfigs, moduleInfrastructure, modules } from '../db/schema';
|
|
70
|
+
import type { ModuleManifest } from '../manifest/schema';
|
|
37
71
|
import { parseStoredConfigValue } from '../services/module-config';
|
|
72
|
+
import { readResolutionContext } from '../variables/context';
|
|
38
73
|
|
|
39
74
|
export async function loadHookConfigMap(
|
|
40
75
|
moduleId: string,
|
|
@@ -51,19 +86,82 @@ export async function loadHookConfigMap(
|
|
|
51
86
|
configMap[c.key] = parseStoredConfigValue(c);
|
|
52
87
|
}
|
|
53
88
|
|
|
54
|
-
|
|
89
|
+
applyMachineAddressFallback(configMap, moduleId, db);
|
|
90
|
+
await applyRecomputedDerivedValues(configMap, moduleId, db);
|
|
91
|
+
|
|
92
|
+
return configMap;
|
|
93
|
+
}
|
|
94
|
+
|
|
95
|
+
/**
|
|
96
|
+
* Fill in the declared variables the config rows have no value for, using the
|
|
97
|
+
* value the resolution context computes right now. Mutates `configMap`.
|
|
98
|
+
*/
|
|
99
|
+
async function applyRecomputedDerivedValues(
|
|
100
|
+
configMap: Record<string, unknown>,
|
|
101
|
+
moduleId: string,
|
|
102
|
+
db: DbClient,
|
|
103
|
+
): Promise<void> {
|
|
104
|
+
const module = db.select().from(modules).where(eq(modules.id, moduleId)).get();
|
|
105
|
+
if (!module?.manifestData) return;
|
|
106
|
+
|
|
107
|
+
const declared = (module.manifestData as ModuleManifest).variables?.owns ?? [];
|
|
108
|
+
const missing = declared.filter((variable) => !(variable.name in configMap));
|
|
109
|
+
if (missing.length === 0) return;
|
|
110
|
+
|
|
111
|
+
let context: Awaited<ReturnType<typeof readResolutionContext>>;
|
|
112
|
+
try {
|
|
113
|
+
context = await readResolutionContext(moduleId, db);
|
|
114
|
+
} catch (error) {
|
|
115
|
+
// Derivation THROWS when a `required: true` variable cannot be resolved —
|
|
116
|
+
// correct at generate time, where a deploy that cannot resolve a required
|
|
117
|
+
// value should stop. This reader also serves health checks and
|
|
118
|
+
// `module run-hook`, which previously could not fail this way at all: a
|
|
119
|
+
// stored row cannot throw. Letting it propagate would mean a provider that
|
|
120
|
+
// is paused, removed, or not yet deployed takes down the health checks of
|
|
121
|
+
// every module that derives from it, reporting healthy consumers as
|
|
122
|
+
// broken.
|
|
123
|
+
//
|
|
124
|
+
// So the hook falls back to what the table holds — exactly what it
|
|
125
|
+
// received before recomputation existed, never less. Logged rather than
|
|
126
|
+
// swallowed (Rule 6.2): a derive that cannot resolve is worth knowing
|
|
127
|
+
// about even when the hook survives it.
|
|
128
|
+
console.error(
|
|
129
|
+
`Could not recompute derived config for '${moduleId}'; the hook sees only its stored config. ` +
|
|
130
|
+
`Derived values (${missing.map((v) => v.name).join(', ')}) may be missing:`,
|
|
131
|
+
error,
|
|
132
|
+
);
|
|
133
|
+
return;
|
|
134
|
+
}
|
|
135
|
+
|
|
136
|
+
for (const variable of missing) {
|
|
137
|
+
const value = context.selfConfig[variable.name];
|
|
138
|
+
if (value !== undefined) {
|
|
139
|
+
configMap[variable.name] = value;
|
|
140
|
+
}
|
|
141
|
+
}
|
|
142
|
+
}
|
|
143
|
+
|
|
144
|
+
/**
|
|
145
|
+
* Fill `target_ip` / `ip.primary` from the deployment machine when the config
|
|
146
|
+
* rows carry no address of their own. Mutates `configMap` in place.
|
|
147
|
+
*/
|
|
148
|
+
function applyMachineAddressFallback(
|
|
149
|
+
configMap: Record<string, unknown>,
|
|
150
|
+
moduleId: string,
|
|
151
|
+
db: DbClient,
|
|
152
|
+
): void {
|
|
153
|
+
if (configMap.target_ip) return;
|
|
55
154
|
|
|
56
155
|
const infra = db
|
|
57
156
|
.select()
|
|
58
157
|
.from(moduleInfrastructure)
|
|
59
158
|
.where(eq(moduleInfrastructure.moduleId, moduleId))
|
|
60
159
|
.get();
|
|
61
|
-
if (!infra?.machineId) return
|
|
160
|
+
if (!infra?.machineId) return;
|
|
62
161
|
|
|
63
162
|
const machine = db.select().from(machines).where(eq(machines.id, infra.machineId)).get();
|
|
64
|
-
if (!machine) return
|
|
163
|
+
if (!machine) return;
|
|
65
164
|
|
|
66
165
|
configMap.target_ip = machine.ipAddress;
|
|
67
166
|
configMap['ip.primary'] = machine.ipAddress;
|
|
68
|
-
return configMap;
|
|
69
167
|
}
|