@celilo/cli 0.20.0 → 0.21.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CELILO_CORE_MODULES.md +2 -2
- package/CELILO_SUBSYSTEMS.md +4 -2
- package/drizzle/0020_dns_registrations_drop_ip.sql +25 -0
- package/drizzle/0021_dns_registration_consumers.sql +63 -0
- package/drizzle/0022_dns_registrations_companion.sql +15 -0
- package/drizzle/0023_public_dns_evidence.sql +19 -0
- package/drizzle/meta/_journal.json +29 -1
- package/package.json +2 -2
- package/schemas/system_config.json +22 -11
- package/src/cli/commands/dns.ts +8 -4
- package/src/cli/commands/events.ts +4 -1
- package/src/cli/commands/system-audit.ts +15 -0
- package/src/cli/commands/system-migrate.test.ts +25 -4
- package/src/cli/commands/system-update.ts +5 -0
- package/src/cli/tui/audit-state.ts +2 -0
- package/src/db/dns-registrations-migration.test.ts +205 -0
- package/src/db/schema.ts +77 -8
- package/src/hooks/define-hook.test.ts +3 -3
- package/src/hooks/executor.test.ts +58 -0
- package/src/hooks/executor.ts +67 -7
- package/src/hooks/run-named-hook.ts +7 -1
- package/src/hooks/test-fixtures/silent-hook.ts +20 -0
- package/src/module/packaging/build.ts +14 -0
- package/src/services/alerting/builtin-monitors.ts +3 -0
- package/src/services/alerting/builtin-source.ts +23 -0
- package/src/services/audit/index.test.ts +2 -0
- package/src/services/audit/index.ts +3 -0
- package/src/services/audit/public-dns-source.ts +55 -0
- package/src/services/audit/public-dns.test.ts +209 -0
- package/src/services/audit/public-dns.ts +286 -0
- package/src/services/audit/types.ts +1 -0
- package/src/services/dns-registrations.test.ts +78 -16
- package/src/services/dns-registrations.ts +107 -19
- package/src/services/fleet-checks.test.ts +47 -1
- package/src/services/fleet-checks.ts +36 -4
- package/src/services/module-subscriptions.test.ts +9 -0
- package/src/services/public-dns-probe.test.ts +81 -0
- package/src/services/public-dns-probe.ts +156 -0
- package/src/services/update/orchestrator.test.ts +2 -0
package/CELILO_CORE_MODULES.md
CHANGED
|
@@ -25,12 +25,12 @@ Each entry: `module id` — what it is — **provides** / **requires** capabilit
|
|
|
25
25
|
- **iptables** — iptables firewall + NAT; cross-VLAN port exposure with recursive upstream delegation. Converge model: `exposeService` registers into the shared-core port-forward registry, then renders the complete ruleset and applies it atomically via `iptables-restore` (default-DROP FORWARD + coarse zone-tier matrix; SSH-free). **provides:** `firewall`.
|
|
26
26
|
- **knot-unbound-internal** — split-horizon internal DNS via Knot (authoritative) + Unbound (recursive); lightweight, plain apt, no .NET. **provides:** `dns_internal`. Ships a base-module-aspect (`modules/knot-unbound-internal/base-module-aspect/`).
|
|
27
27
|
- **technitium** — internal split-horizon DNS resolver + authoritative server (web UI + HTTP API); heavier alternative to knot-unbound. **provides:** `dns_internal`. Ships a base-module-aspect (`modules/technitium/base-module-aspect/`).
|
|
28
|
-
- **namecheap** — public DNS A-record management via Namecheap Dynamic DNS API (HTTP, no browser automation). **provides:** `dns_registrar`.
|
|
28
|
+
- **namecheap** — public DNS A-record management via Namecheap Dynamic DNS API (HTTP, no browser automation). A caller supplies a NAME and nothing else: the address is the source IP of celilo's own update, re-derived on every assert. Registering `<domain>` also claims `www.<domain>` and vice versa (best effort, reported back as `outputs.companion_fqdn` so the framework's `public_dns` check watches it — Namecheap answers `ErrCount 0` for `www` updates it does not apply). DDNS passwords are keyed by the **registrable domain**, never the FQDN. **provides:** `dns_registrar`.
|
|
29
29
|
- **wireguard** — owns the admin WireGuard tunnel on the firewall host: interface, listen port, peers (as records), and client subnet are module config rather than hand-maintained state. Exposes the listen port and **registers the client subnet as a trusted source**, so VPN reach into the managed zones is in the firewall registry and every converge re-emits it. Records `network.control-plane-vpn.subnet`, which the internal resolver's split-horizon view also consumes. Adopts a running tunnel in place (existing key and peers retained; `wg syncconf`, never `wg-quick down`) because that tunnel is the operator's recovery path. **requires:** `firewall` (and the provider must support trusted-source registration — `iptables` does, `greenwave` does not).
|
|
30
30
|
|
|
31
31
|
## Public edge (ingress / identity)
|
|
32
32
|
|
|
33
|
-
- **caddy** — reverse proxy with automatic HTTPS (HTTP-01 ACME); the standard HTTPS ingress. **provides:** `public_web`. **requires:** `dns_registrar`, `firewall`.
|
|
33
|
+
- **caddy** — reverse proxy with automatic HTTPS (HTTP-01 ACME); the standard HTTPS ingress. Emits a 301 redirect block for each served name's `www`/apex companion (kept out of the served-hostname set, so a companion never blocks the ACME wait). Its `health_check`'s DNS item is `internal_dns_resolution` — it digs from the management host behind the split-horizon resolver and is evidence about the in-fleet view only; public reachability is the framework's `public_dns` check. **provides:** `public_web`. **requires:** `dns_registrar`, `firewall`.
|
|
34
34
|
- **generic-cpanel-hosting-provider** — publishes static sites into a subfolder of a domain on a cPanel/SSH web host celilo does **not** govern (the host owns the domain, DNS, TLS and web server). Systemless like **namecheap** — no `requires.system`, no zone, no IPAM; it holds credentials for an external party. One provider serves many accounts, resolved by hostname; onboarding takes the account password once, `ssh-copy-id`s celilo's key, then discards it. **provides:** `external_web`.
|
|
35
35
|
- **authentik** — Authentik identity provider with OIDC (Docker Compose: server, worker, Postgres, Redis). **provides:** `idp`. **requires:** `public_web`, `dns_registrar`, `firewall`.
|
|
36
36
|
|
package/CELILO_SUBSYSTEMS.md
CHANGED
|
@@ -38,6 +38,7 @@ see `openspec/specs/`. Companion doc: [CELILO_CORE_MODULES.md](./CELILO_CORE_MOD
|
|
|
38
38
|
- **Canonical capability registry** — `packages/capabilities/src/capability-registry.ts` — `KNOWN_CAPABILITY_NAMES` (the authoritative list), `CapabilityRegistry` type. Public surface: `packages/capabilities/src/index.ts`.
|
|
39
39
|
- **Loader (wires provider factories into hook contexts)** — `apps/celilo/src/hooks/capability-loader.ts` — `loadCapabilityFunctions`, `resolveFirewallNatIp`.
|
|
40
40
|
- **Ledger wrappers (stateful capabilities)** — `apps/celilo/src/services/dns-registrations.ts` (`withDnsRegistrationLedger`), `apps/celilo/src/services/dns-internal-records.ts` (`withDnsInternalLedger`).
|
|
41
|
+
- **Public DNS ledger (`dns_registrations` + `dns_registration_consumers`)** — `apps/celilo/src/services/dns-registrations.ts` — records WHICH MODULE ASKED FOR WHICH NAME, and deliberately **no address**: the published value is source-detected on every assert and observable on demand from public DNS, and a stored copy is one careless read away from becoming an instruction again (celilo#626 — the 15-minute `refresh_registrations` replayed addresses written by an older celilo and took five public names dark for nine days while every in-fleet check stayed green). Consumers are a SET, not a column: the FK cascade on a single `consumer_module_id` decided when a LIVE record was forgotten, so a row now survives until its LAST consumer is removed (`openspec/changes/public-dns-reachability/design.md` D5). Orphaned rows are pruned on read (`pruneOrphanedRegistrations`) rather than by trigger — SQLite fires delete triggers for FK-cascaded deletes only with `recursive_triggers` on. `companion: true` marks a name celilo claimed on a module's behalf (`www.<domain>` ↔ `<domain>`) rather than one a module asked for; the provider reports it as `outputs.companion_fqdn` whether or not the claim reported success, because a provider's success response is not evidence of publication. Operator surface: `celilo dns registrations`.
|
|
41
42
|
|
|
42
43
|
### Known capabilities (impl → provider module)
|
|
43
44
|
|
|
@@ -45,7 +46,7 @@ see `openspec/specs/`. Companion doc: [CELILO_CORE_MODULES.md](./CELILO_CORE_MOD
|
|
|
45
46
|
|---|---|---|
|
|
46
47
|
| `public_web` | `packages/capabilities/src/public-web.ts` (`createPublicWeb`). **Route lifecycle** is framework-owned at both ends, not per-module: removing a consumer runs `apps/celilo/src/services/web-route-cleanup.ts` (delete rows → reclaim `/srv/www/<slug>` → emit `routes_changed`) whether or not the module has an `on_uninstall`; deploying a provider runs `apps/celilo/src/services/public-web-republish.ts` (re-run every static consumer's `on_install`) so a provider rebuild refills the web roots it destroyed. | caddy |
|
|
47
48
|
| `idp` | `packages/capabilities/src/idp.ts` | authentik |
|
|
48
|
-
| `dns_registrar` | `packages/capabilities/src/dns-registrar.ts` (`registerHost`) | namecheap |
|
|
49
|
+
| `dns_registrar` | `packages/capabilities/src/dns-registrar.ts` (`registerHost({ fqdn })` — **there is no way to supply an address**; the provider re-derives it from its own update's source IP on every assert) | namecheap |
|
|
49
50
|
| `external_web` | `packages/capabilities/src/external-web.ts` (`publishStaticSite`). **OFF-FLEET static publishing** — the counterpart to `public_web`. See the in-fleet vs off-fleet note below. | generic-cpanel-hosting-provider |
|
|
50
51
|
| `firewall` | `packages/capabilities/src/firewall.ts` (`exposeService`, `unexposeService`, `listExposedServices`). **Converge model**: register into the shared-core port-forward registry → render the complete ruleset → apply atomically (see the firewall converge note below). | greenwave, iptables |
|
|
51
52
|
| `dns_internal` | `packages/capabilities/src/dns-internal.ts` | knot-unbound-internal, technitium |
|
|
@@ -198,8 +199,9 @@ is currently wrong, and routes carry the message to a person's phone. Design:
|
|
|
198
199
|
- **Alert identity** — `apps/celilo/src/services/alerting/keys.ts` — the key grammar (`module:<id>[/check:<name>]`, `builtin:<check>[/<kind>:<target>]`) that makes "the same problem" the same alert across runs. `moduleAlertKey`, `moduleCheckAlertKey`, `builtinAlertKey`, `parseAlertKey`.
|
|
199
200
|
- **Reconciliation** — `apps/celilo/src/services/alerting/reconcile.ts` (`reconcile`) — a successful run's failing-key set is authoritative and resolution is by SET DIFFERENCE (absent ⇒ resolved). A run whose outcome is `error` resolves NOTHING and fires a module-level alert instead: the false-all-clear guard.
|
|
200
201
|
- **Monitor execution** — `apps/celilo/src/services/alerting/run-monitor.ts` (`runOneMonitor`) + `sweep.ts` (`selectDueMonitors`) + `builtin-monitors.ts` / `health-coverage.ts` (the built-in checks and the "module with no health check" coverage check).
|
|
201
|
-
- **Scheduled audit categories (`builtin_check` monitors)** — `apps/celilo/src/services/alerting/builtin-source.ts` — `SCHEDULABLE_BUILTIN_CHECKS` is the list of `celilo system audit` categories cheap enough to run every sweep: `machines_reachable`, `backups`, `disk_space`, and `
|
|
202
|
+
- **Scheduled audit categories (`builtin_check` monitors)** — `apps/celilo/src/services/alerting/builtin-source.ts` — `SCHEDULABLE_BUILTIN_CHECKS` is the list of `celilo system audit` categories cheap enough to run every sweep: `machines_reachable`, `backups`, `disk_space`, `abandoned_operations`, and `public_dns` (`apps/celilo/src/services/audit/abandoned-operations.ts` — ≥3 abandonments of the same (module, operation) in 7d, the fingerprint of an operation being killed mid-flight). Everything else in the audit needs the whole world injected (proxmox, terraform, registry) and is not schedulable. Enable one with `celilo monitor add backups --interval 1h`. Targets are tab-completable — `completion.ts` reads `SCHEDULABLE_BUILTIN_CHECKS` directly rather than a hand-copied list, so a newly-schedulable check is completable immediately.
|
|
202
203
|
- **Disk-space check** — `apps/celilo/src/services/audit/disk-space.ts` (`auditDiskSpace`, pure over measurements) + `apps/celilo/src/services/disk-probe.ts` (`probeDiskUsage`). Thresholds: `drift` at 85%, `blocked` at 95% — early enough to act on, since a check that fires at exhaustion reports an outage rather than preventing one. ⚠️ **The local management box is MEASURED, not exempted.** `probeMachines()` deliberately reports the local box reachable without probing it (celilo has no SSH key for itself, and the question is meaningless there); copying that shortcut into a disk check would skip the host most likely to fill — the one that stages backups, caches modules and writes the logs, and the one that DID fill. Local reads `statfs`; remote runs `df -P /` over the same bounded SSH. `percentUsed` matches `df`'s capacity semantics (excludes root-reserved blocks) so an alert and an operator's own `df` agree. An unmeasurable host yields a `todo` finding — recorded, never paged, because `machines_reachable` is already paging for that host. Findings are subjected on the **hostname**, not the machine UUID, because suppression resolves a machine's ancestor key from the hostname (see #596, where `machines_reachable` gets this wrong and its alerts therefore never suppress anything). The `backups` roster comes from `apps/celilo/src/services/audit/backup-source.ts` (`loadBackupAuditInfo`), shared with `celilo system audit` so both judge the same fleet.
|
|
204
|
+
- **Public-DNS reachability check (the only check with an OFF-FLEET vantage)** — `apps/celilo/src/services/audit/public-dns.ts` (`auditPublicDns`, pure over an injected probe and the previous run's counters) + `apps/celilo/src/services/public-dns-probe.ts` (the probe) + `audit/public-dns-source.ts` (ledger names + the `public_dns_evidence` counters). Every other check in celilo looks from INSIDE, behind a split-horizon resolver that deliberately answers with an in-zone address — correct for its purpose, and why all of them reported healthy for the nine days of celilo#626. This one resolves every `dns_registrations` FQDN through an **off-fleet resolver** (`public_dns.resolver`, default `1.1.1.1`) and compares it against the address the fleet appears to come from per an independent **echo service** (`public_dns.echo_url`, default `https://api.ipify.org`). Three properties are load-bearing: `assertOffFleetResolver` REFUSES a resolver matching `dns.primary`/`dns.fallback` (a check that quietly used the fleet's resolver would pass forever — the original bug one layer up); the expectation never comes from the registrar's own response (self-agreement, and Namecheap returns `ErrCount 0` for `www` updates it does not apply); and a divergence is a finding only once it OUTLIVES the record's own TTL, measured from the last assert, or it would page on every ISP re-lease. Missing evidence is counted rather than read as success — one undetermined run is silent, N consecutive ones are their own finding (`public_dns_evidence`), which is the hole celilo-website's isitup.org probe demonstrated live. Codes: `public_dns_stale`, `public_dns_missing`, `public_dns_companion_unclaimed`, `public_dns_unverifiable`. Spec: `openspec/specs/public-dns-reachability/spec.md`.
|
|
203
205
|
- **The sweep** — `apps/celilo/src/services/alerting/sweep-runner.ts` (`runSweep`) — the ordered pass that makes alerting run by itself: run due monitors → promote past-grace alerts → re-evaluate suppression → flush quiet-hours deferrals → notify. Driven by `celilo alerts sweep` on `timer.tick.5m`. Never throws for one bad monitor.
|
|
204
206
|
- **Suppression (topology-derived, never configured)** — `apps/celilo/src/services/alerting/suppression.ts` — `ancestorKeysFor`/`findSuppressor`/`machineAlertKey`. A firing machine explains its modules' failures; a firing capability provider explains its zone's consumers. Derived from `module_systems`, so it cannot drift from reality. Deploy windows: `deploy-hooks.ts` (`openDeployWindow`/`closeDeployWindows` — closed by module, so a crashed deploy self-heals).
|
|
205
207
|
- **Escalation & quiet hours** — `escalation.ts` (`decideEscalation`, every reason to stay silent enumerated) + `quiet-hours.ts` (`isWithinQuietHours`, Intl-based and DST-safe). Quiet hours defer the MESSAGE while the escalation clock keeps running.
|
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
-- Drop `dns_registrations.ip`.
|
|
2
|
+
--
|
|
3
|
+
-- THIS MIGRATION IS THE SWEEP, not the tidying after one.
|
|
4
|
+
--
|
|
5
|
+
-- A row's stored `ip` was replayed by `refresh_registrations` every 15
|
|
6
|
+
-- minutes as the address to publish. Rows written before #464/#466 carry the
|
|
7
|
+
-- external address the caller passed at the time, so once the ISP re-leased,
|
|
8
|
+
-- the refresh re-published a dead address on a timer, indefinitely, reporting
|
|
9
|
+
-- success every tick. celilo#626: five public names dark for nine days,
|
|
10
|
+
-- including apt.celilo.computer and the module registry, while every in-fleet
|
|
11
|
+
-- check stayed green because the split-horizon resolver answers with a
|
|
12
|
+
-- reachable internal address.
|
|
13
|
+
--
|
|
14
|
+
-- Fixing only the write path leaves every armed legacy row armed until
|
|
15
|
+
-- somebody happens to redeploy the module that owns it. Dropping the column
|
|
16
|
+
-- disarms all of them at once, and makes the whole class unrepresentable
|
|
17
|
+
-- rather than merely unlikely — there is no longer a value to go stale.
|
|
18
|
+
--
|
|
19
|
+
-- The address is not migrated anywhere. It is observable on demand from
|
|
20
|
+
-- public DNS (that is what the new `public_dns` check does), and a stored
|
|
21
|
+
-- copy is one careless read away from becoming an instruction again.
|
|
22
|
+
--
|
|
23
|
+
-- Plain DROP COLUMN: `ip` is in no index and no FOREIGN KEY clause.
|
|
24
|
+
|
|
25
|
+
ALTER TABLE `dns_registrations` DROP COLUMN `ip`;
|
|
@@ -0,0 +1,63 @@
|
|
|
1
|
+
-- Replace `dns_registrations.consumer_module_id` with a consumer SET.
|
|
2
|
+
--
|
|
3
|
+
-- The column was overwritten by every re-assert, and it is not a label: the
|
|
4
|
+
-- FK cascade on it decides when a LIVE record is forgotten. Recovering the
|
|
5
|
+
-- fleet with `run-hook caddy on_install` re-attributed auth.lunacycle.net,
|
|
6
|
+
-- buildyourowninternet.dev, dj-incarnadine.org and peterbanka.org to caddy,
|
|
7
|
+
-- so removing caddy would have cascade-deleted registrations that authentik
|
|
8
|
+
-- and the site modules still serve. Keep-first has the mirror failure: if
|
|
9
|
+
-- module A introduces a name and B later needs it too, removing A drops the
|
|
10
|
+
-- row out from under B. Both encode a single owner where there may be several.
|
|
11
|
+
--
|
|
12
|
+
-- With a set, the row survives until its LAST consumer is removed — which is
|
|
13
|
+
-- also the only basis on which "can this record be deleted at the registrar?"
|
|
14
|
+
-- could ever be answered. See design.md D5.
|
|
15
|
+
--
|
|
16
|
+
-- Existing rows are seeded from the single value they carry. The pre-recovery
|
|
17
|
+
-- attribution for the four names above is NOT recoverable — it was overwritten
|
|
18
|
+
-- in place before this migration existed. Seeding preserves what the table
|
|
19
|
+
-- actually holds rather than pretending to reconstruct what it used to.
|
|
20
|
+
--
|
|
21
|
+
-- Order matters: `dns_registrations` is rebuilt (SQLite refuses to drop a
|
|
22
|
+
-- column named in a FOREIGN KEY clause) and `foreign_keys` is ON, so a child
|
|
23
|
+
-- table created before the rebuild would be cascade-emptied by the DROP.
|
|
24
|
+
-- The seed is parked in an FK-free scratch table across it.
|
|
25
|
+
|
|
26
|
+
CREATE TABLE `__dns_consumer_seed` (
|
|
27
|
+
`registration_id` integer NOT NULL,
|
|
28
|
+
`module_id` text NOT NULL,
|
|
29
|
+
`first_seen_at` integer NOT NULL
|
|
30
|
+
);
|
|
31
|
+
--> statement-breakpoint
|
|
32
|
+
INSERT INTO `__dns_consumer_seed` SELECT `id`, `consumer_module_id`, `registered_at` FROM `dns_registrations`;
|
|
33
|
+
--> statement-breakpoint
|
|
34
|
+
CREATE TABLE `__new_dns_registrations` (
|
|
35
|
+
`id` integer PRIMARY KEY AUTOINCREMENT NOT NULL,
|
|
36
|
+
`provider_module_id` text NOT NULL,
|
|
37
|
+
`fqdn` text NOT NULL,
|
|
38
|
+
`registered_at` integer DEFAULT (unixepoch()) NOT NULL,
|
|
39
|
+
`refreshed_at` integer,
|
|
40
|
+
FOREIGN KEY (`provider_module_id`) REFERENCES `modules`(`id`) ON UPDATE no action ON DELETE cascade
|
|
41
|
+
);
|
|
42
|
+
--> statement-breakpoint
|
|
43
|
+
INSERT INTO `__new_dns_registrations` SELECT `id`, `provider_module_id`, `fqdn`, `registered_at`, `refreshed_at` FROM `dns_registrations`;--> statement-breakpoint
|
|
44
|
+
DROP TABLE `dns_registrations`;--> statement-breakpoint
|
|
45
|
+
ALTER TABLE `__new_dns_registrations` RENAME TO `dns_registrations`;--> statement-breakpoint
|
|
46
|
+
CREATE UNIQUE INDEX `dns_registrations_provider_fqdn_idx` ON `dns_registrations` (`provider_module_id`,`fqdn`);
|
|
47
|
+
--> statement-breakpoint
|
|
48
|
+
CREATE TABLE `dns_registration_consumers` (
|
|
49
|
+
`id` integer PRIMARY KEY AUTOINCREMENT NOT NULL,
|
|
50
|
+
`registration_id` integer NOT NULL,
|
|
51
|
+
`module_id` text NOT NULL,
|
|
52
|
+
`first_seen_at` integer DEFAULT (unixepoch()) NOT NULL,
|
|
53
|
+
FOREIGN KEY (`registration_id`) REFERENCES `dns_registrations`(`id`) ON UPDATE no action ON DELETE cascade,
|
|
54
|
+
FOREIGN KEY (`module_id`) REFERENCES `modules`(`id`) ON UPDATE no action ON DELETE cascade
|
|
55
|
+
);
|
|
56
|
+
--> statement-breakpoint
|
|
57
|
+
CREATE UNIQUE INDEX `dns_registration_consumers_unique_idx` ON `dns_registration_consumers` (`registration_id`,`module_id`);
|
|
58
|
+
--> statement-breakpoint
|
|
59
|
+
INSERT INTO `dns_registration_consumers` (`registration_id`, `module_id`, `first_seen_at`)
|
|
60
|
+
SELECT `registration_id`, `module_id`, `first_seen_at` FROM `__dns_consumer_seed`
|
|
61
|
+
WHERE `module_id` IN (SELECT `id` FROM `modules`);
|
|
62
|
+
--> statement-breakpoint
|
|
63
|
+
DROP TABLE `__dns_consumer_seed`;
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
-- `dns_registrations.companion` — the name celilo claimed on a module's
|
|
2
|
+
-- behalf rather than one a module asked for.
|
|
3
|
+
--
|
|
4
|
+
-- `www.<domain>` and `<domain>` are one name pair: declaring either claims the
|
|
5
|
+
-- other. The declared name fails the deploy if it cannot be published; the
|
|
6
|
+
-- companion is best effort, because failing a deploy over a name nobody
|
|
7
|
+
-- requested is punitive. Best effort must not mean silent, though — a working
|
|
8
|
+
-- www beside a dead apex is celilo#626 in miniature — so companion rows are
|
|
9
|
+
-- watched by the `public_dns` check like any other, and carry the manual
|
|
10
|
+
-- registrar remediation when they diverge.
|
|
11
|
+
--
|
|
12
|
+
-- Existing rows are all declared names: nothing claimed a companion before
|
|
13
|
+
-- this migration.
|
|
14
|
+
|
|
15
|
+
ALTER TABLE `dns_registrations` ADD COLUMN `companion` integer DEFAULT false NOT NULL;
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
-- `public_dns_evidence` — consecutive runs the public-reachability check could
|
|
2
|
+
-- obtain no evidence about a subject.
|
|
3
|
+
--
|
|
4
|
+
-- A probe that could not run is not a pass. celilo-website's external check
|
|
5
|
+
-- already got this half right and half wrong: it records an unreachable prober
|
|
6
|
+
-- as *undetermined* rather than a failure (correct — a prober blip once paged
|
|
7
|
+
-- three modules at once) but then emits no check item at all, so a real outage
|
|
8
|
+
-- produced complete silence for nine days.
|
|
9
|
+
--
|
|
10
|
+
-- Counting the absences closes that without reintroducing the noise: one
|
|
11
|
+
-- undetermined run is still silent, N consecutive ones are their own finding,
|
|
12
|
+
-- distinguishable from "the site is unreachable". A subject that answers has
|
|
13
|
+
-- its row dropped, which is what makes the count consecutive.
|
|
14
|
+
|
|
15
|
+
CREATE TABLE `public_dns_evidence` (
|
|
16
|
+
`subject` text PRIMARY KEY NOT NULL,
|
|
17
|
+
`undetermined_runs` integer DEFAULT 0 NOT NULL,
|
|
18
|
+
`last_checked_at` integer DEFAULT (unixepoch()) NOT NULL
|
|
19
|
+
);
|
|
@@ -141,6 +141,34 @@
|
|
|
141
141
|
"when": 1783500000000,
|
|
142
142
|
"tag": "0019_backup_pid",
|
|
143
143
|
"breakpoints": true
|
|
144
|
+
},
|
|
145
|
+
{
|
|
146
|
+
"idx": 20,
|
|
147
|
+
"version": "6",
|
|
148
|
+
"when": 1783600000000,
|
|
149
|
+
"tag": "0020_dns_registrations_drop_ip",
|
|
150
|
+
"breakpoints": true
|
|
151
|
+
},
|
|
152
|
+
{
|
|
153
|
+
"idx": 21,
|
|
154
|
+
"version": "6",
|
|
155
|
+
"when": 1783700000000,
|
|
156
|
+
"tag": "0021_dns_registration_consumers",
|
|
157
|
+
"breakpoints": true
|
|
158
|
+
},
|
|
159
|
+
{
|
|
160
|
+
"idx": 22,
|
|
161
|
+
"version": "6",
|
|
162
|
+
"when": 1783800000000,
|
|
163
|
+
"tag": "0022_dns_registrations_companion",
|
|
164
|
+
"breakpoints": true
|
|
165
|
+
},
|
|
166
|
+
{
|
|
167
|
+
"idx": 23,
|
|
168
|
+
"version": "6",
|
|
169
|
+
"when": 1783900000000,
|
|
170
|
+
"tag": "0023_public_dns_evidence",
|
|
171
|
+
"breakpoints": true
|
|
144
172
|
}
|
|
145
173
|
]
|
|
146
|
-
}
|
|
174
|
+
}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@celilo/cli",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.21.0",
|
|
4
4
|
"description": "Celilo — home lab orchestration CLI",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"bin": {
|
|
@@ -57,7 +57,7 @@
|
|
|
57
57
|
},
|
|
58
58
|
"dependencies": {
|
|
59
59
|
"@aws-sdk/client-s3": "^3.1024.0",
|
|
60
|
-
"@celilo/capabilities": "^0.
|
|
60
|
+
"@celilo/capabilities": "^1.0.0",
|
|
61
61
|
"@celilo/cli-display": "^0.1.10",
|
|
62
62
|
"@celilo/core": "^0.5.0",
|
|
63
63
|
"@celilo/event-bus": "^0.3.0",
|
|
@@ -13,7 +13,7 @@
|
|
|
13
13
|
"network.dmz.subnet": {
|
|
14
14
|
"type": "string",
|
|
15
15
|
"pattern": "^\\d{1,3}\\.\\d{1,3}\\.\\d{1,3}\\.\\d{1,3}/\\d{1,2}$",
|
|
16
|
-
"description": "DMZ subnet CIDR (not defaulted
|
|
16
|
+
"description": "DMZ subnet CIDR (not defaulted \u2014 appears only when a firewall module provides this zone; see openspec/specs/progressive-zone-disclosure/spec.md)"
|
|
17
17
|
},
|
|
18
18
|
"network.dmz.gateway": {
|
|
19
19
|
"type": "string",
|
|
@@ -29,7 +29,7 @@
|
|
|
29
29
|
"network.app.subnet": {
|
|
30
30
|
"type": "string",
|
|
31
31
|
"pattern": "^\\d{1,3}\\.\\d{1,3}\\.\\d{1,3}\\.\\d{1,3}/\\d{1,2}$",
|
|
32
|
-
"description": "App subnet CIDR (not defaulted
|
|
32
|
+
"description": "App subnet CIDR (not defaulted \u2014 appears only when a firewall module provides this zone)"
|
|
33
33
|
},
|
|
34
34
|
"network.app.gateway": {
|
|
35
35
|
"type": "string",
|
|
@@ -45,7 +45,7 @@
|
|
|
45
45
|
"network.secure.subnet": {
|
|
46
46
|
"type": "string",
|
|
47
47
|
"pattern": "^\\d{1,3}\\.\\d{1,3}\\.\\d{1,3}\\.\\d{1,3}/\\d{1,2}$",
|
|
48
|
-
"description": "Secure subnet CIDR (not defaulted
|
|
48
|
+
"description": "Secure subnet CIDR (not defaulted \u2014 appears only when a firewall module provides this zone)"
|
|
49
49
|
},
|
|
50
50
|
"network.secure.gateway": {
|
|
51
51
|
"type": "string",
|
|
@@ -61,12 +61,12 @@
|
|
|
61
61
|
"network.internal.subnet": {
|
|
62
62
|
"type": "string",
|
|
63
63
|
"pattern": "^\\d{1,3}\\.\\d{1,3}\\.\\d{1,3}\\.\\d{1,3}/\\d{1,2}$",
|
|
64
|
-
"description": "Internal subnet CIDR (home devices, trusted; not defaulted
|
|
64
|
+
"description": "Internal subnet CIDR (home devices, trusted; not defaulted \u2014 discovered from the management box's primary interface at celilo-mgmt install)"
|
|
65
65
|
},
|
|
66
66
|
"network.internal.gateway": {
|
|
67
67
|
"type": "string",
|
|
68
68
|
"pattern": "^\\d{1,3}\\.\\d{1,3}\\.\\d{1,3}\\.\\d{1,3}$",
|
|
69
|
-
"description": "Internal gateway IP address (not defaulted
|
|
69
|
+
"description": "Internal gateway IP address (not defaulted \u2014 discovered from the default route)"
|
|
70
70
|
},
|
|
71
71
|
"network.internal.vlan": {
|
|
72
72
|
"type": "integer",
|
|
@@ -77,17 +77,17 @@
|
|
|
77
77
|
"network.control-plane-vpn.subnet": {
|
|
78
78
|
"type": "string",
|
|
79
79
|
"pattern": "^\\d{1,3}\\.\\d{1,3}\\.\\d{1,3}\\.\\d{1,3}/\\d{1,2}$",
|
|
80
|
-
"description": "WireGuard VPN client subnet CIDR (remote-access tunnel; consumed by technitium's split-horizon
|
|
80
|
+
"description": "WireGuard VPN client subnet CIDR (remote-access tunnel; consumed by technitium's split-horizon \u2192 in-zone caddy view, #275, and by the firewall's trusted sources once the wireguard module registers it)"
|
|
81
81
|
},
|
|
82
82
|
"network.secure-mgmt.subnet": {
|
|
83
83
|
"type": "string",
|
|
84
84
|
"pattern": "^\\d{1,3}\\.\\d{1,3}\\.\\d{1,3}\\.\\d{1,3}/\\d{1,2}$",
|
|
85
|
-
"description": "Control-plane subnet CIDR
|
|
85
|
+
"description": "Control-plane subnet CIDR \u2014 the network celilo-mgr itself occupies when it does not sit on the internal LAN (not defaulted \u2014 discovered at celilo-mgmt install, or recorded by firewall onboarding). Derives the firewall's trusted sources and the internal resolver's split-horizon view for celilo's own traffic."
|
|
86
86
|
},
|
|
87
87
|
"network.secure-mgmt.gateway": {
|
|
88
88
|
"type": "string",
|
|
89
89
|
"pattern": "^\\d{1,3}\\.\\d{1,3}\\.\\d{1,3}\\.\\d{1,3}$",
|
|
90
|
-
"description": "Control-plane gateway IP address (not defaulted
|
|
90
|
+
"description": "Control-plane gateway IP address (not defaulted \u2014 discovered from the default route)"
|
|
91
91
|
},
|
|
92
92
|
"network.secure-mgmt.vlan": {
|
|
93
93
|
"type": "integer",
|
|
@@ -97,20 +97,31 @@
|
|
|
97
97
|
},
|
|
98
98
|
"firewall.trusted_subnets": {
|
|
99
99
|
"type": "string",
|
|
100
|
-
"description": "Operator-declared trusted source subnets (comma-separated CIDRs) permitted to reach every managed zone. ADDITIVE to the control-plane subnet celilo derives and to the sources modules register
|
|
100
|
+
"description": "Operator-declared trusted source subnets (comma-separated CIDRs) permitted to reach every managed zone. ADDITIVE to the control-plane subnet celilo derives and to the sources modules register \u2014 it cannot drop control-plane trust, which would block the SSH every hook and converge runs over."
|
|
101
101
|
},
|
|
102
102
|
"dns.primary": {
|
|
103
103
|
"type": "string",
|
|
104
104
|
"pattern": "^\\d{1,3}\\.\\d{1,3}\\.\\d{1,3}\\.\\d{1,3}$",
|
|
105
|
-
"description": "Primary DNS server IP address (not defaulted
|
|
105
|
+
"description": "Primary DNS server IP address (not defaulted \u2014 discovered from the host resolver at celilo-mgmt install; repointed at a dns_internal provider when one deploys)"
|
|
106
106
|
},
|
|
107
107
|
"dns.fallback": {
|
|
108
108
|
"type": "string",
|
|
109
|
-
"description": "Fallback DNS servers (comma-separated IP addresses; not defaulted
|
|
109
|
+
"description": "Fallback DNS servers (comma-separated IP addresses; not defaulted \u2014 discovered, with 1.1.1.1 as last resort)"
|
|
110
110
|
},
|
|
111
111
|
"ssh.public_key": {
|
|
112
112
|
"type": "string",
|
|
113
113
|
"description": "SSH public key for container access"
|
|
114
|
+
},
|
|
115
|
+
"public_dns.resolver": {
|
|
116
|
+
"type": "string",
|
|
117
|
+
"pattern": "^\\d{1,3}\\.\\d{1,3}\\.\\d{1,3}\\.\\d{1,3}$",
|
|
118
|
+
"default": "1.1.1.1",
|
|
119
|
+
"description": "Resolver the public_dns check asks what the INTERNET resolves for the fleet's names. Must not be a resolver the fleet itself uses \u2014 the fleet resolver runs split-horizon and would make the check pass whatever the public internet sees (celilo#626). celilo refuses a value matching dns.primary / dns.fallback."
|
|
120
|
+
},
|
|
121
|
+
"public_dns.echo_url": {
|
|
122
|
+
"type": "string",
|
|
123
|
+
"default": "https://api.ipify.org",
|
|
124
|
+
"description": "Echo service reporting the address the fleet appears to come from \u2014 the expectation public DNS is compared against. Deliberately not the registrar's own response, which is self-agreement (and which Namecheap returns successfully for updates it does not apply)."
|
|
114
125
|
}
|
|
115
126
|
},
|
|
116
127
|
"additionalProperties": true
|
package/src/cli/commands/dns.ts
CHANGED
|
@@ -3,7 +3,7 @@
|
|
|
3
3
|
*
|
|
4
4
|
* `registrations` lists the dns_registrations ledger: every (provider,
|
|
5
5
|
* fqdn) the framework has registered via dns_registrar.registerHost,
|
|
6
|
-
* with
|
|
6
|
+
* with every module that depends on it and when it was last re-asserted
|
|
7
7
|
* by the provider's refresh_registrations hook. Read-only; names only
|
|
8
8
|
* (module ids), never UUIDs. See
|
|
9
9
|
* designs/DISPATCHER_DAEMON_AND_TIMER_EVENTS.md (B2b).
|
|
@@ -40,12 +40,16 @@ export async function handleDnsRegistrations(
|
|
|
40
40
|
}
|
|
41
41
|
|
|
42
42
|
const now = new Date();
|
|
43
|
-
|
|
43
|
+
// No address column: the ledger stores none. What a name actually resolves
|
|
44
|
+
// to publicly is what `celilo system audit`'s public_dns check reports, from
|
|
45
|
+
// off-fleet — reading it back out of celilo's own table is what made a
|
|
46
|
+
// nine-day outage invisible (design.md D1).
|
|
47
|
+
const header = ['FQDN', 'KIND', 'PROVIDER', 'CONSUMERS', 'REGISTERED', 'REFRESHED'];
|
|
44
48
|
const table = rows.map((r) => [
|
|
45
49
|
r.fqdn,
|
|
46
|
-
r.
|
|
50
|
+
r.companion ? 'companion' : 'declared',
|
|
47
51
|
r.providerModuleId,
|
|
48
|
-
r.
|
|
52
|
+
r.consumerModuleIds.join(', ') || '—',
|
|
49
53
|
formatAge(r.registeredAt, now),
|
|
50
54
|
formatAge(r.refreshedAt, now),
|
|
51
55
|
]);
|
|
@@ -152,7 +152,10 @@ export async function handleEventsRunHook(args: string[]): Promise<CommandResult
|
|
|
152
152
|
};
|
|
153
153
|
|
|
154
154
|
const logger = createConsoleLogger(moduleId, sub.hook);
|
|
155
|
-
const result = await runNamedHook(moduleId, sub.hook as HookName, db, logger, {
|
|
155
|
+
const result = await runNamedHook(moduleId, sub.hook as HookName, db, logger, {
|
|
156
|
+
inputs,
|
|
157
|
+
timeoutMs: sub.timeout_ms,
|
|
158
|
+
});
|
|
156
159
|
|
|
157
160
|
if (result.notDefined) {
|
|
158
161
|
return { success: false, error: `Module '${moduleId}' declares no '${sub.hook}' hook to run` };
|
|
@@ -46,6 +46,10 @@ import {
|
|
|
46
46
|
} from '../../services/audit/cli-version';
|
|
47
47
|
import type { MachineReachableResult } from '../../services/audit/machines-reachable';
|
|
48
48
|
import type { ModuleVersionFetcher } from '../../services/audit/module-versions';
|
|
49
|
+
import {
|
|
50
|
+
loadPublicDnsEvidence,
|
|
51
|
+
loadPublicDnsRecords,
|
|
52
|
+
} from '../../services/audit/public-dns-source';
|
|
49
53
|
import { makeJournalReader, readAppliedMigrations } from '../../services/audit/schema';
|
|
50
54
|
import type { SecretCheckResult } from '../../services/audit/secrets-decryptable';
|
|
51
55
|
import type { ServiceCredentialsResult } from '../../services/audit/services-credentials';
|
|
@@ -56,6 +60,7 @@ import { collectFirewallReach } from '../../services/firewall-reach';
|
|
|
56
60
|
import { runAllHealthChecks } from '../../services/health-runner';
|
|
57
61
|
import { probeMachines } from '../../services/machine-probe';
|
|
58
62
|
import { parseStoredConfigValue } from '../../services/module-config';
|
|
63
|
+
import { createPublicDnsProbe, loadPublicDnsProbeSettings } from '../../services/public-dns-probe';
|
|
59
64
|
import { buildTerraformEnvForModule } from '../../services/terraform-env';
|
|
60
65
|
import { hasFlag } from '../parser';
|
|
61
66
|
import type { CommandResult } from '../types';
|
|
@@ -381,6 +386,16 @@ async function buildAuditDeps(onProgress?: (msg: string) => void) {
|
|
|
381
386
|
secretsDecryptable: { results: secretResults },
|
|
382
387
|
servicesReachable: { results: serviceReachableResults },
|
|
383
388
|
machinesReachable: { results: machineReachableResults },
|
|
389
|
+
// The only check here whose vantage point is OUTSIDE the fleet. Its
|
|
390
|
+
// undetermined counters are read but not written from this path: an
|
|
391
|
+
// operator-run audit is not a run of a schedule, so counting it towards
|
|
392
|
+
// "N consecutive runs found no evidence" would misreport how long the
|
|
393
|
+
// fleet has been unverifiable. The monitor sweep owns that (D2).
|
|
394
|
+
publicDns: {
|
|
395
|
+
records: loadPublicDnsRecords(db),
|
|
396
|
+
probe: createPublicDnsProbe(loadPublicDnsProbeSettings(db)),
|
|
397
|
+
evidence: loadPublicDnsEvidence(db),
|
|
398
|
+
},
|
|
384
399
|
// Reads the record the poller already writes — this check never performs a
|
|
385
400
|
// read of its own. One that did would drain the queue and eat the
|
|
386
401
|
// acknowledgement it exists to protect (#541).
|
|
@@ -1,11 +1,28 @@
|
|
|
1
1
|
import { Database } from 'bun:sqlite';
|
|
2
2
|
import { afterEach, beforeEach, describe, expect, it } from 'bun:test';
|
|
3
|
-
import { mkdtempSync, rmSync } from 'node:fs';
|
|
3
|
+
import { mkdtempSync, readFileSync, rmSync } from 'node:fs';
|
|
4
4
|
import { tmpdir } from 'node:os';
|
|
5
5
|
import { join } from 'node:path';
|
|
6
|
-
import { closeDb } from '../../db/client';
|
|
6
|
+
import { closeDb, findMigrationsFolder } from '../../db/client';
|
|
7
7
|
import { handleSystemMigrate } from './system-migrate';
|
|
8
8
|
|
|
9
|
+
/**
|
|
10
|
+
* The newest migration, read from the journal rather than written down here.
|
|
11
|
+
*
|
|
12
|
+
* These assertions used to name `0019_backup_pid` literally, so every
|
|
13
|
+
* subsequent migration broke a test that has nothing to do with it. What is
|
|
14
|
+
* under test is that `--status` REPORTS the head and any gap below it, not
|
|
15
|
+
* which migration happens to be head today.
|
|
16
|
+
*/
|
|
17
|
+
function latestMigrationTag(): string {
|
|
18
|
+
const journal = JSON.parse(
|
|
19
|
+
readFileSync(join(findMigrationsFolder(), 'meta', '_journal.json'), 'utf8'),
|
|
20
|
+
) as { entries: { tag: string }[] };
|
|
21
|
+
const tag = journal.entries.at(-1)?.tag;
|
|
22
|
+
if (!tag) throw new Error('Migration journal is empty');
|
|
23
|
+
return tag;
|
|
24
|
+
}
|
|
25
|
+
|
|
9
26
|
describe('handleSystemMigrate', () => {
|
|
10
27
|
let dir: string;
|
|
11
28
|
|
|
@@ -51,7 +68,7 @@ describe('handleSystemMigrate', () => {
|
|
|
51
68
|
expect(result.success).toBe(true);
|
|
52
69
|
if (result.success) {
|
|
53
70
|
expect(result.message).toMatch(/Applied migrations: \d+/);
|
|
54
|
-
expect(result.message).toContain(
|
|
71
|
+
expect(result.message).toContain(latestMigrationTag());
|
|
55
72
|
expect(result.message).toContain('Pending: none');
|
|
56
73
|
expect(result.message).toContain('columns');
|
|
57
74
|
}
|
|
@@ -61,6 +78,7 @@ describe('handleSystemMigrate', () => {
|
|
|
61
78
|
await handleSystemMigrate();
|
|
62
79
|
closeDb();
|
|
63
80
|
// Rewind one migration, the way an upgrade that never ran would look.
|
|
81
|
+
const head = latestMigrationTag();
|
|
64
82
|
const raw = new Database(process.env.CELILO_DB_PATH as string);
|
|
65
83
|
raw.run(
|
|
66
84
|
'DELETE FROM `__drizzle_migrations` WHERE created_at = (SELECT MAX(created_at) FROM `__drizzle_migrations`)',
|
|
@@ -75,7 +93,10 @@ describe('handleSystemMigrate', () => {
|
|
|
75
93
|
|
|
76
94
|
expect(result.success).toBe(false);
|
|
77
95
|
if (!result.success) {
|
|
78
|
-
|
|
96
|
+
// The rewound migration is named as pending…
|
|
97
|
+
expect(result.error).toContain(head);
|
|
98
|
+
// …and the dropped COLUMN is reported independently, which is the
|
|
99
|
+
// thing a table count cannot see (celilo#604).
|
|
79
100
|
expect(result.error).toContain('backups.pid');
|
|
80
101
|
}
|
|
81
102
|
|
|
@@ -28,6 +28,7 @@ import { RegistryClient } from '../../registry/client';
|
|
|
28
28
|
import { runAudit } from '../../services/audit';
|
|
29
29
|
import { loadAbandonedOperations } from '../../services/audit/abandoned-operations';
|
|
30
30
|
import { fetchLatestCliVersion } from '../../services/audit/cli-version';
|
|
31
|
+
import { unusedPublicDnsProbe } from '../../services/audit/public-dns';
|
|
31
32
|
import { makeJournalReader, readAppliedMigrations } from '../../services/audit/schema';
|
|
32
33
|
import { createModuleBackup, createSystemStateBackup } from '../../services/backup-create';
|
|
33
34
|
import { runAllHealthChecks, runModuleHealthCheck } from '../../services/health-runner';
|
|
@@ -587,6 +588,10 @@ export async function handleSystemUpdate(
|
|
|
587
588
|
secretsDecryptable: { results: [] },
|
|
588
589
|
servicesReachable: { results: [] },
|
|
589
590
|
machinesReachable: { results: [] },
|
|
591
|
+
// Public reachability needs a network round trip per name; the update
|
|
592
|
+
// flow's partial audit does no probing. `system audit` and the scheduled
|
|
593
|
+
// monitor own this check.
|
|
594
|
+
publicDns: { records: [], probe: unusedPublicDnsProbe },
|
|
590
595
|
transportReads: { statuses: [], now: new Date(), staleAfterMs: 30 * 60_000 },
|
|
591
596
|
trustedSources: { firewalls: [] },
|
|
592
597
|
};
|
|
@@ -84,6 +84,7 @@ export const ALL_CATEGORIES: readonly DriftCategory[] = [
|
|
|
84
84
|
'secrets_decryptable',
|
|
85
85
|
'services_reachable',
|
|
86
86
|
'machines_reachable',
|
|
87
|
+
'public_dns',
|
|
87
88
|
'disk_space',
|
|
88
89
|
'transport_reads',
|
|
89
90
|
'trusted_sources',
|
|
@@ -105,6 +106,7 @@ export const CATEGORY_LABELS: Record<DriftCategory, string> = {
|
|
|
105
106
|
secrets_decryptable: 'Secrets',
|
|
106
107
|
services_reachable: 'Service reachability',
|
|
107
108
|
machines_reachable: 'Machine reachability',
|
|
109
|
+
public_dns: 'Public DNS reachability',
|
|
108
110
|
disk_space: 'Disk space',
|
|
109
111
|
transport_reads: 'Transport readability',
|
|
110
112
|
trusted_sources: 'Trusted networks',
|