@celilo/cli 0.16.2 → 0.18.0

package/CELILO_CORE_MODULES.md

@@ -26,7 +26,7 @@ Each entry: `module id` — what it is — **provides** / **requires** capabilit
 - **knot-unbound-internal** — split-horizon internal DNS via Knot (authoritative) + Unbound (recursive); lightweight, plain apt, no .NET. **provides:** `dns_internal`. Ships a base-module-aspect (`modules/knot-unbound-internal/base-module-aspect/`).
 - **technitium** — internal split-horizon DNS resolver + authoritative server (web UI + HTTP API); heavier alternative to knot-unbound. **provides:** `dns_internal`. Ships a base-module-aspect (`modules/technitium/base-module-aspect/`).
 - **namecheap** — public DNS A-record management via Namecheap Dynamic DNS API (HTTP, no browser automation). **provides:** `dns_registrar`.
-- **wireguard** — owns the admin WireGuard tunnel on the firewall host: interface, listen port, peers (as records), and client subnet are module config rather than hand-maintained state. Exposes the listen port and **registers the client subnet as a trusted source**, so VPN reach into the managed zones is in the firewall registry and every converge re-emits it. Records `network.vpn.subnet`, which the internal resolver's split-horizon view also consumes. Adopts a running tunnel in place (existing key and peers retained; `wg syncconf`, never `wg-quick down`) because that tunnel is the operator's recovery path. **requires:** `firewall` (and the provider must support trusted-source registration — `iptables` does, `greenwave` does not).
+- **wireguard** — owns the admin WireGuard tunnel on the firewall host: interface, listen port, peers (as records), and client subnet are module config rather than hand-maintained state. Exposes the listen port and **registers the client subnet as a trusted source**, so VPN reach into the managed zones is in the firewall registry and every converge re-emits it. Records `network.control-plane-vpn.subnet`, which the internal resolver's split-horizon view also consumes. Adopts a running tunnel in place (existing key and peers retained; `wg syncconf`, never `wg-quick down`) because that tunnel is the operator's recovery path. **requires:** `firewall` (and the provider must support trusted-source registration — `iptables` does, `greenwave` does not).
 
 ## Public edge (ingress / identity)
 

package/CELILO_SUBSYSTEMS.md

@@ -92,7 +92,7 @@ Module hooks reach a remote box ONLY through these typed primitives
 - **`runAppCommand` / `runAppCommandWithSecret`** — escape-hatch on-box command; the secret variant feeds the secret on **stdin** (`$SECRET`), never argv.
 - **`streamBackup` / `streamRestore` / `fetchFile` / `pushFile`** — binary-safe streaming via local shell redirect/pipe.
 - **`waitFor`** — predicate-poll combinator.
-- **`applyRenderedConfig`** — converge: write rendered config → validate → apply → rollback (one round-trip). Used by caddy (Caddyfile), knot (views), iptables (ruleset).
+- **`applyRenderedConfig`** — converge: write rendered config → validate → apply → rollback (one round-trip). Used by caddy (Caddyfile), knot (views), iptables (ruleset). On success it RETAINS the file it replaced as `<path>.celilo-prev` — the durable "what celilo last rendered" record a whole-file converge diffs against.
 - **`installAuthorizedKey`** — ONE-TIME credential bootstrap for an off-fleet account: `ssh-copy-id` under a password taken from the child ENV (`SSH_ASKPASS_REQUIRE=force`, so no `sshpass` dependency and the password never lands in a command string). Idempotent; used by `external_web` onboarding so every later publish is key-based.
 - **`tailLog` / `grepLog`** — journald reads (regex / ignoreCase).
 
@@ -112,10 +112,34 @@ runner seam (`execRunner` real / `createMockRunner` for tests) lives in
   (`renderRuleset`): registry + firewall state → a complete `iptables-restore`
   file. Default-DROP FORWARD + established/related + egress + the coarse
   zone-tier matrix (dmz→app, app→secure from `network.<zone>.subnet`) + per-service DNAT allows.
+  Egress permission and egress TRANSLATION are emitted together or neither: a
+  blanket `-o <wan>` MASQUERADE on a leaf, one `-s <subnet> -o <uplink>` rule per
+  network on a DOWNSTREAM firewall (no external edge). An upstream does not
+  translate on a downstream firewall's behalf.
+  `subnetsNeedingTranslation(state)` is that set — EVERY network behind the
+  firewall (tiers ∪ every declared zone subnet ∪ control plane ∪ registered
+  trusted sources) minus whichever of them CONTAINS the egress interface's own
+  address (`egressIp`, read off the box). Not just the data-plane tiers: a
+  downstream firewall's egress leg IS its LAN leg, so an untranslated network is
+  unreachable from a LAN host as well as from the internet.
 - **Converge** — `modules/iptables/scripts/firewall-functions.ts` (`converge`):
-  `exposeService`/`unexposeService` register into the store, then render + apply
-  atomically via `applyRenderedConfig` (`iptables-restore`). Replaces the old
-  per-rule `iptables -A`; the registry (not `iptables -L`) is the source of truth.
+  `exposeService`/`unexposeService` register into the store, then render, CHECK,
+  DIFF, and only then apply atomically via `applyRenderedConfig`
+  (`iptables-restore`). Replaces the old per-rule `iptables -A`; the registry
+  (not `iptables -L`) is the source of truth. It does NOT apply unconditionally:
+  a failing check or an unexplainable removal throws, leaving the working
+  ruleset in place (`config.force` overrides, on the record).
+- **Render-time completeness checks (pure)** — `modules/iptables/scripts/ruleset-checks.ts`
+  (`checkRenderedSet`, `blockingFindings`, `formatConvergeRefusal`): the SEMANTIC
+  counterpart to `iptables-restore --test`, which only validates syntax. Reports
+  a zone permitted to egress with no translation covering it, a control plane
+  that cannot reach a zone it manages, and an unknown control plane (a warning,
+  distinct from "known and absent"). Runs under `config.dryRun` too.
+- **Pre-apply ruleset diff (pure)** — `modules/iptables/scripts/ruleset-diff.ts`
+  (`parseRuleset`, `diffRuleset`, `driftAgainstLive`, `snapshotCommand`,
+  `parseSnapshot`): what the converge would remove, and whether a registry change
+  explains it. Exact set algebra against the retained previous render, not a
+  heuristic. Live-vs-persisted drift is reported, never decided on.
 - **Trusted-source registry (desired state)** — `trusted_sources` DB table
   (migration `0016`) + `apps/celilo/src/services/trusted-sources.ts`
   (`buildTrustedSourceStore`, `composeTrustedSubnets`). The sibling primitive to
@@ -144,7 +168,7 @@ runner seam (`execRunner` real / `createMockRunner` for tests) lives in
 - **Deploy pipeline** — `apps/celilo/src/services/module-deploy.ts`.
 - **DNS provider backfill** (re-emit registrations when a provider deploys) — `apps/celilo/src/services/dns-provider-backfill.ts` — `isDnsInternalProvider`, `backfillProviderDns`.
 - **Base-module aspects (fan-out across the fleet)** — `apps/celilo/src/services/aspect-runner.ts` — `planAspectFanOut`, `runAspectFanOut`, `maybeRunAspectForTrigger`. Aspect content lives in `modules/<m>/base-module-aspect/` (e.g. knot-unbound-internal, technitium).
-- **In-flight operation lock** — `apps/celilo/src/services/module-operations.ts` — `startOperation`/`completeOperation`/`failOperation` record deploy/uninstall/backup/restore in `module_operations`; `refuseIfInFlight`/`checkInFlight` are what backup and restore consult. Deploy and uninstall REGISTER but never check: it is a one-way guard protecting backup/restore consistency, not a general mutex (`openspec/specs/management-server-backup/spec.md` "In-flight operation refusal"). A row stops holding the lock once it is GONE, STOPPED/zombie (`isPidRunnable`, `ps -o state=` — `kill(pid,0)` calls a Ctrl-Z'd process alive), or older than `OPERATION_TTL_MS` (2h). The TTL is not redundancy: a pid is a recycled number, and once the pid space wraps an old row names an unrelated healthy process. Operator surface: `celilo module operations [list|clear] [--all]` (`apps/celilo/src/cli/commands/module-operations.ts`).
+- **In-flight operation lock** — `apps/celilo/src/services/module-operations.ts` — `startOperation`/`completeOperation`/`failOperation` record deploy/uninstall/backup/restore in `module_operations`; `refuseIfInFlight`/`checkInFlight` are what backup and restore consult. Deploy and uninstall REGISTER but never check: it is a one-way guard protecting backup/restore consistency, not a general mutex (`openspec/specs/management-server-backup/spec.md` "In-flight operation refusal"). A row stops holding the lock once it is GONE, STOPPED/zombie (`isPidRunnable`, `ps -o state=` — `kill(pid,0)` calls a Ctrl-Z'd process alive), or older than `OPERATION_TTL_MS` (2h). The TTL is not redundancy: a pid is a recycled number, and once the pid space wraps an old row names an unrelated healthy process. Operator surface: `celilo module operations [list|clear] [--abandoned] [--all]` (`apps/celilo/src/cli/commands/module-operations.ts`); `list` shows only what holds the lock, abandoned rows are summarised unless `--abandoned`. Abandoned rows are reclaimed hourly by the `celilo-operations-sweep` bus subscriber (`timer.tick.1h` → `celilo module operations clear`, armed by `ensureOperationsSweepSubscriber` from module registration and `celilo system migrate`). `clear` MARKS rows failed rather than deleting them, and that is load-bearing: the `abandoned_operations` audit reads exactly those released rows to notice one module's operation dying over and over.
 - **Backup artifact encryption** — `apps/celilo/src/services/backup-cipher.ts` — `encryptFileToFile`/`decryptFileToFile`, file-in/file-out and streamed, used by every backup writer (`backup-create.ts`) and reader (`backup-restore.ts`, `restore-from-file.ts`). Do NOT route artifacts through `secrets/encryption.ts`: that API is string-in/string-out for short DB values, and feeding it a tar cost base64 (1.33x) then hex (2x) then `JSON.stringify` — ~9x the artifact in memory, which OOM-killed forgejo's 774 MB backup, and a hard ~805 MB ceiling from the max string length that no amount of RAM raises. On-disk format is `magic "CELILOBK" (8) | version (1) | iv (16) | ciphertext | GCM tag (16)`; the tag is a trailer because it does not exist until the last byte is encrypted. `decryptFileToFile` still reads the pre-2026-07 JSON-envelope artifacts, discriminating on the magic bytes — the envelope's own `schemaVersion` cannot serve, as it lives inside the encrypted tar.
 
 ## Generation & templating
@@ -174,7 +198,8 @@ is currently wrong, and routes carry the message to a person's phone. Design:
 - **Alert identity** — `apps/celilo/src/services/alerting/keys.ts` — the key grammar (`module:<id>[/check:<name>]`, `builtin:<check>[/<kind>:<target>]`) that makes "the same problem" the same alert across runs. `moduleAlertKey`, `moduleCheckAlertKey`, `builtinAlertKey`, `parseAlertKey`.
 - **Reconciliation** — `apps/celilo/src/services/alerting/reconcile.ts` (`reconcile`) — a successful run's failing-key set is authoritative and resolution is by SET DIFFERENCE (absent ⇒ resolved). A run whose outcome is `error` resolves NOTHING and fires a module-level alert instead: the false-all-clear guard.
 - **Monitor execution** — `apps/celilo/src/services/alerting/run-monitor.ts` (`runOneMonitor`) + `sweep.ts` (`selectDueMonitors`) + `builtin-monitors.ts` / `health-coverage.ts` (the built-in checks and the "module with no health check" coverage check).
-- **Scheduled audit categories (`builtin_check` monitors)** — `apps/celilo/src/services/alerting/builtin-source.ts` — `SCHEDULABLE_BUILTIN_CHECKS` is the list of `celilo system audit` categories cheap enough to run every sweep: `machines_reachable` and `backups`. Everything else in the audit needs the whole world injected (proxmox, terraform, registry) and is not schedulable. Enable one with `celilo monitor add backups --interval 1h`. The `backups` roster comes from `apps/celilo/src/services/audit/backup-source.ts` (`loadBackupAuditInfo`), shared with `celilo system audit` so both judge the same fleet.
+- **Scheduled audit categories (`builtin_check` monitors)** — `apps/celilo/src/services/alerting/builtin-source.ts` — `SCHEDULABLE_BUILTIN_CHECKS` is the list of `celilo system audit` categories cheap enough to run every sweep: `machines_reachable`, `backups`, `disk_space`, and `abandoned_operations` (`apps/celilo/src/services/audit/abandoned-operations.ts` — ≥3 abandonments of the same (module, operation) in 7d, the fingerprint of an operation being killed mid-flight). Everything else in the audit needs the whole world injected (proxmox, terraform, registry) and is not schedulable. Enable one with `celilo monitor add backups --interval 1h`. Targets are tab-completable — `completion.ts` reads `SCHEDULABLE_BUILTIN_CHECKS` directly rather than a hand-copied list, so a newly-schedulable check is completable immediately.
+- **Disk-space check** — `apps/celilo/src/services/audit/disk-space.ts` (`auditDiskSpace`, pure over measurements) + `apps/celilo/src/services/disk-probe.ts` (`probeDiskUsage`). Thresholds: `drift` at 85%, `blocked` at 95% — early enough to act on, since a check that fires at exhaustion reports an outage rather than preventing one. ⚠️ **The local management box is MEASURED, not exempted.** `probeMachines()` deliberately reports the local box reachable without probing it (celilo has no SSH key for itself, and the question is meaningless there); copying that shortcut into a disk check would skip the host most likely to fill — the one that stages backups, caches modules and writes the logs, and the one that DID fill. Local reads `statfs`; remote runs `df -P /` over the same bounded SSH. `percentUsed` matches `df`'s capacity semantics (excludes root-reserved blocks) so an alert and an operator's own `df` agree. An unmeasurable host yields a `todo` finding — recorded, never paged, because `machines_reachable` is already paging for that host. Findings are subjected on the **hostname**, not the machine UUID, because suppression resolves a machine's ancestor key from the hostname (see #596, where `machines_reachable` gets this wrong and its alerts therefore never suppress anything). The `backups` roster comes from `apps/celilo/src/services/audit/backup-source.ts` (`loadBackupAuditInfo`), shared with `celilo system audit` so both judge the same fleet.
 - **The sweep** — `apps/celilo/src/services/alerting/sweep-runner.ts` (`runSweep`) — the ordered pass that makes alerting run by itself: run due monitors → promote past-grace alerts → re-evaluate suppression → flush quiet-hours deferrals → notify. Driven by `celilo alerts sweep` on `timer.tick.5m`. Never throws for one bad monitor.
 - **Suppression (topology-derived, never configured)** — `apps/celilo/src/services/alerting/suppression.ts` — `ancestorKeysFor`/`findSuppressor`/`machineAlertKey`. A firing machine explains its modules' failures; a firing capability provider explains its zone's consumers. Derived from `module_systems`, so it cannot drift from reality. Deploy windows: `deploy-hooks.ts` (`openDeployWindow`/`closeDeployWindows` — closed by module, so a crashed deploy self-heals).
 - **Escalation & quiet hours** — `escalation.ts` (`decideEscalation`, every reason to stay silent enumerated) + `quiet-hours.ts` (`isWithinQuietHours`, Intl-based and DST-safe). Quiet hours defer the MESSAGE while the escalation clock keeps running.
@@ -194,9 +219,12 @@ Creation, scheduling and freshness. A module declares an `on_backup` hook and a
 
 - **Creation** — `apps/celilo/src/services/backup-create.ts` — `createModuleBackup` (invokes the module's `on_backup` hook into an encrypted envelope), `createSystemStateBackup` (celilo.db), `findBackupEligibleModules`, `isBackupDue`. Storage destinations: `backup-storage.ts`. Retention: `backup-retention.ts` (`pruneBackupsForModule`). Restore: `backup-restore.ts`.
 - **Cadence (one accessor)** — `apps/celilo/src/services/backup-schedule.ts` — `effectiveBackupSchedule(manifest)`. An absent `backup.schedule` means `daily`, NOT `manual`; opting out takes an explicit `manual`. Both the freshness audit and the backup sweep must read cadence through this one function, or a module can be alerted-on but never backed up.
-- **The sweep** — `apps/celilo/src/services/backup-sweep.ts` — `runBackupSweep` (the pass that makes backups run by themselves: for each eligible module, is its declared cadence due → back it up → apply declared retention) + `ensureBackupSweepSubscriber`. Driven by `celilo backup sweep` on `timer.tick.1h` — the coarsest tick that can still serve an `hourly` cadence. Armed automatically: `registerModuleSubscriptions` registers the subscriber for any module declaring an `on_backup` hook, so it appears on install or `module update`. A run refused by the in-flight operation lock is a skip retried next tick, never a failure.
+- **The sweep** — `apps/celilo/src/services/backup-sweep.ts` — `runBackupSweep` (the pass that makes backups run by themselves: reclaim orphaned staging → for each eligible module, is its declared cadence due → back it up → apply declared retention) + `ensureBackupSweepSubscriber`. Driven by `celilo backup sweep` on `timer.tick.1h` — the coarsest tick that can still serve an `hourly` cadence. Armed from BOTH `registerModuleSubscriptions` (any module declaring an `on_backup` hook, so it appears on install or `module update`) AND `celilo system migrate`, which the `.deb` postinst runs on every apt upgrade — registering only from module install/update meant a corrected budget never reached an existing fleet, since the row already exists and module updates can be weeks apart. A run refused by the in-flight operation lock is a skip retried next tick, never a failure.
+- **⚠️ The sweep's budget is stated, never inherited** — `BACKUP_SWEEP_TIMEOUT_MS` (4h) and `BACKUP_SWEEP_MAX_ATTEMPTS` (1) in `backup-sweep.ts`. The event bus defaults to `timeout_ms: 60000` / `max_attempts: 3`, and inheriting them made scheduled backups structurally impossible: one forgejo backup measured ~5.5 minutes (1.3 GB result, 3.9 GB peak staging), so the dispatcher SIGTERMed it at 60s — three times an hour, for days. The retry count is half the bug, not a detail: an impossible pass retried 3x strands 3x the staging (27 GB in 5.7 hours on celilo-mgr). The hourly tick IS the retry.
+- **Staging reclamation** — `apps/celilo/src/services/backup-staging.ts` — `reapOrphanedStaging`, `stagingDirFor` (the single source of truth for `/tmp/celilo-backup-<record.id>`, shared with `backup-create.ts` so writer and reaper cannot drift). `backup-create.ts` removes its staging in a `finally`, which is correct and NOT enough: a `finally` never runs when the process is killed by a signal — dispatcher timeout, OOM, Ctrl-C, reboot — and those strand the LARGEST directories. So reclamation is kill-mode agnostic by construction: it asks "is anyone still using this?", answered from the `backups.pid` column plus `isPidRunnable`, both of which outlive the process. A directory is removed only when its owner is provably gone (record absent, record terminal, pid dead, or past `STAGING_TTL_MS` = 6h — the TTL is the only check surviving pid reuse). A live backup is always kept; an unrecognised name is ignored, never deleted. Reclaiming a record that still claimed `in_progress` also marks it failed with `ABANDONED_BACKUP_MESSAGE`, so `celilo backup list` stops showing phantom in-flight backups and the `backups` drift check cannot read a dead attempt as a fresh backup.
 - **Freshness audit** — `apps/celilo/src/services/audit/backups.ts` (`auditBackups`) — `backup_missing` / `backup_stale` drift findings against the same declared cadence.
 - **CLI** — `apps/celilo/src/cli/commands/` — `backup-sweep.ts`, `backup-create.ts` (also `celilo module backup`), `backup-list.ts`, `backup-restore.ts`, `backup-prune.ts`, `backup-delete.ts`, `backup-import.ts`, `backup-pull.ts`, `backup-name.ts`.
+- **Storage destinations CLI** — `storage-add-local.ts`, `storage-add-s3.ts`, `storage-list.ts`, `storage-verify.ts`, `storage-set-default.ts`, `storage-set-path.ts` (relocate a local destination, migrating existing archives unless `--no-migrate`), `storage-remove.ts`. Any credential change goes through `updateStorageCredentials` in `backup-storage.ts`, which clears the verification stamp — a `✓ Verified` must never describe a destination it was not measured against (#566).
 
 ## Persistence
 
@@ -204,7 +232,7 @@ Creation, scheduling and freshness. A module declares an `on_backup` hook and a
 
 ## Events
 
-- **Event bus** — `packages/event-bus/src/index.ts` — `Bus`, `openBus`, `defineEvents`, `defineHandler`, `runDispatcher`, pattern matching + timer ticks (`emitDueTimerTicks`, `retentionSweep`).
+- **Event bus** — `packages/event-bus/src/index.ts` — `Bus`, `openBus`, `defineEvents`, `defineHandler`, `runDispatcher`, pattern matching + timer ticks (`emitDueTimerTicks`, `retentionSweep`). **Exactly one dispatcher per bus**: `runDispatcher` refuses to start while another dispatcher's process is alive (`assertSoleDispatcher` in `dispatcher.ts`, liveness via `bus.liveDispatchers()` — `kill(pid,0)`, not heartbeat age, since a tick blocks for as long as its slowest handler). The exclusion is here rather than in the systemd unit because a stranded dispatcher can sit outside the unit's cgroup where `KillMode` cannot reach it (#580). `bus.health()` reports `dispatcherCount`/`dispatchers` and a `duplicate_dispatcher` status; `checkDispatcher` (`services/fleet-checks.ts`) fails on more than one.
 
 ## Remote API (drive the CLI over the wire)
 
@@ -218,7 +246,8 @@ Run any celilo command on celilo-mgr over SSH instead of screen-scraping `ssh <h
 - **Access control** — `apps/celilo/src/services/api-access.ts` — `grantPrincipal`, `isAuthorized` (deny-by-default, `command:subcommand` grants), `renderAuthorizedKeys`. Table: `api_principals` (`apps/celilo/src/db/schema.ts`). CLI: `apps/celilo/src/cli/commands/api.ts` (`api grant|list|revoke|authorized-keys|key new`).
 - **Mid-run interview bridge (`kind:daemon` responder)** — `apps/celilo/src/services/remote-responder.ts` — `startRemoteResponder` bridges bus `interview.required.*` ↔ wire.
 - **Server provisioning** — the `celilo-bootstrap` deb (`packaging/celilo-bootstrap/scripts/postinst`) creates the non-root `celilo-api` landing account + sshd; membership in the `celilo` group + `/etc/sudoers.d/celilo` (`!use_pty`) gives api-serve DB access via the wrapper's sudo-drop.
-- **Self-upgrade (apt)** — `celilo apt-upgrade` (`apps/celilo/src/cli/commands/apt-upgrade.ts`) upgrades the deb-installed `celilo`/`celilo-bootstrap` packages (`apt-get update` → `--only-upgrade install`) then spawns a fresh `celilo system migrate` (ISS-0100). It's the RW target behind the MCP's registry-derived `celilo_apt_upgrade` tool; the celilo user's two apt invocations are scoped-sudo'd by `/etc/sudoers.d/celilo-apt-upgrade`, shipped by `celilo-bootstrap`.
+- **Self-upgrade (apt)** — `celilo apt-upgrade` (`apps/celilo/src/cli/commands/apt-upgrade.ts`) upgrades the deb-installed `celilo`/`celilo-bootstrap` packages (`apt-get update` → `--only-upgrade install`) then spawns a fresh `celilo system migrate` (ISS-0100). It's the RW target behind the MCP's registry-derived `celilo_apt_upgrade` tool; the celilo user's two apt invocations are scoped-sudo'd by `/etc/sudoers.d/celilo-apt-upgrade`, shipped by `celilo-bootstrap`. **This upgrades celilo ITSELF — not the modules it manages. For those, see Module auto-upgrade below; the two are routinely confused.**
+- **Module auto-upgrade (registry-poll CD)** — the *pull* half of continuous deployment: celilo-mgr polls the registry and upgrades opted-in modules unattended. Spec: `openspec/specs/module-auto-upgrade/spec.md`. Entry points: `apps/celilo/src/cli/commands/module-upgrade.ts` — `runRegistryPoll` (the `--poll` path), `selectPollTargets` (pure: `autoUpgrade && latest && change ∉ {up-to-date, ahead}`), `upgradeOneModule` (update → backup → deploy → verify), `needsPreUpgradeBackup`, `pickAutoUpgrade`/`pickUpgradePolicy` (both fail closed/safe); `classifyVersionChange` in `module-update.ts` (treats a registry `+N` revision as a patch); `resolveDeployPosture` in `apps/celilo/src/services/deploy-posture.ts`. Trigger: celilo-mgmt's `registry-poll` subscription (`modules/celilo-mgmt/manifest.yml`) on `timer.tick.15m` with handler **`celilo module upgrade --poll`** — the flag is REQUIRED, since the dispatcher appends the event id positionally and a bare handler would consume it as the optional module name (silent: 3108 deliveries, 0 successes). Operator controls are framework config keys settable on ANY module (`FRAMEWORK_CONFIG_KEYS` in `module-config.ts`): `auto_upgrade` (opt-in, default false) and `upgrade_policy` (`by-semver`|`always-safe`|`always-fast`), validated at set time because both readers fail open. ⚠️ `always-safe` guarantees safe *posture*, NOT a backup — `needsPreUpgradeBackup` also requires the TARGET manifest to declare an `on_backup` hook, else it warns and proceeds. Confirm a data-bearing module declares `on_backup` before enabling `auto_upgrade` on it. The *build* half (app CI publishing a `.netapp` on merge) is not yet shipped — `openspec/changes/build-bus-poll-cd`.
 - **MCP service (`@celilo/mcp`)** — `packages/mcp/src/` — an operator-facing stdio MCP server (official `@modelcontextprotocol/sdk`, bin `celilo-mcp`) that drives a remote celilo server over the Remote API for an AI client. Two-item config (`config.ts`: `server` + `defaultUser`, env or `~/.config/celilo-mcp/config.json`). Dual-principal auth (`auth.ts`: `celilo-mcp auth setup` enrolls read-only `celilo-mcp-ro` + full `celilo-mcp-rw` ed25519 keypairs, prints the exact `celilo api grant` lines the operator runs server-side). Transport (`transport.ts`): reuses `@celilo/core` `runRemoteClient`, selecting the principal by `ssh -i <key>` and capturing structured output. Tool surface is generated LIVE from the server's command registry — `registry-fetch.ts` fetches `celilo commands --json` (+ `service list --json` for configured providers) over the RO principal on connect; `tools-from-registry.ts` (pure) projects that into one tool per runnable leaf, grouped by top-level command (`celilo_module_*`, `celilo_proxmox_*`, …), each with a Zod input schema from the leaf's args/flags and a read/write tag → RO/RW routing, plus a generic `celilo_run` escape hatch. Auto-detect hides provider-gated groups (e.g. `celilo_proxmox_*` until a Proxmox service is configured) and re-detects on a timer, emitting `notifications/tools/list_changed` when the surface changes. Coverage gate (`tests/coverage.test.ts`) asserts every registry leaf maps to a tool. Composite RO troubleshooting tools (`troubleshoot.ts` pure correlation + `troubleshoot-tools.ts` thin adapters): `celilo_assess_module <id>` and `celilo_fleet_status` correlate `celilo audit --json` (the drift backbone) with the `module list --json` roster into a per-module / fleet-wide verdict. Design: `openspec/changes/celilo-mcp-service/proposal.md`. (Distinct from the dev/ops `@celilo/mcp-server` below.)
 
 ## E2E simulation
@@ -227,4 +256,5 @@ Run any celilo command on celilo-mgr over SSH instead of screen-scraping `ssh <h
 - **Run wrapper** — `infra/scripts/cele2e-run.sh` lives in the separate `infra/` clone, **not** in this repo. See the cele2e section of the repo-root `CLAUDE.md` for the operator workflow.
 - **Signal simulators** — three containers, three jobs. `docker/Dockerfile.signal-cli` runs the REAL unlinked daemon (`network().withSignalCli()`, reachable at `signal-cli.lab`) so `e2e/tests/signal-contract.test.ts` can re-check celilo's understanding of the JSON-RPC surface against the actual binary — the pebble pattern. `docker/Dockerfile.signal-sim` runs `simulators/signal-cli/server.ts` (`withSignalSim()`, `signal-sim.lab`), the drivable stand-in with a control surface (`/_control/inbound`, `/_control/sent`, `/_control/unlink`). `/_control/inbound` from the LINKED account's own number emits a `syncMessage.sentMessage` transcript rather than a `dataMessage`, because that is the only shape the real daemon delivers a note-to-self in — and a note-to-self is what every reply is in the default single-operator setup (#460). `docker/Dockerfile.signal-release` (`withSignalRelease()`, `signal-release.lab`) serves the signal-cli release tarball so the module's deploy-time download resolves inside the sealed network — the download is served, never skipped. The libsignal aarch64 native is compiled at `build-infra` time from the module's own recipe (`packages/e2e/scripts/stage-libsignal.ts` → `modules/signal/build/`) and staged into the apt-repo pool, so the recipe is exercised for real on every rebuild while the deploy stays fast and the network stays sealed.
 - **Public-boundary NAT model** — `packages/e2e/config/routing/` — exactly ONE NAT sits between the fleet and the simulated internet: the customer firewall (`fw-main` in `direct-internet`, `fw-isp` in the two-layer default), which MASQUERADEs to `100.100.0.100`. The ISP edge `fw-ext` ROUTES the customer's `100.100.0.0/24` and must never re-NAT it (`-s 100.100.0.0/24 -j RETURN` ahead of its MASQUERADE) — an ISP does not NAT a subscriber that already holds a public address. This is load-bearing, not cosmetic: Namecheap-style DDNS registers the SOURCE address when the caller omits `ip=`, which is how celilo registers public names since #464/#466, so a second NAT here publishes the simulator's own address for every public hostname and quietly breaks ACME, inbound reach, and seeded apex records. Constants: `externalWanIp()` / `externalWanSubnet()` in `src/types.ts`. The corollary: every simulator on `internet-external` must default-route via fw-ext (`100.64.0.1`) so it can reply to the customer's public address — Docker's bridge gateway has no path across networks. `config/routing/public-sim-entrypoint.sh` is the shared two-liner; the DNS hierarchy, pebble, isitup and celilo-website-sim already carried it, and npm-registry / registry / apt-repo / minio / cpanel-host only appeared to work because the double NAT put fw-ext's on-link address in the source field.
+- **Simulated address plan** — `packages/e2e/src/types.ts` — `SIM_PRIVATE_SUPERNET` (`10.226.0.0/16`), `zoneIp(zone, host)`, `ZONE_SUBNETS`, `ZONE_GATEWAYS`. Every simulated PRIVATE zone is derived from this one table; the compose generator, `zone-classifier.ts` and the harness's `system init` all read it, so renumbering the whole sim is a one-line change. The sim deliberately does NOT reuse a real fleet's zone /24s (#539): the previous plan was byte-identical to production's, so a stack leaked on celilo's own forgejo-builder — which lives in the real dmz — claimed the builder's own subnet and blackholed every containerized CI job's route to the forge for ~135s. Teardown cannot prevent that (a SIGKILL runs no handler), so the addresses moved instead; the second, better reason is that a suite which only passes on production's exact octets is asserting one site's address plan rather than celilo's behaviour. `src/address-plan.test.ts` is the recurrence gate — it fails if a zone leaves the supernet, or if any retired fleet prefix reappears anywhere in `packages/e2e`, `e2e/tests` or `modules/*/e2e`.
 - **MCP server (agent-driven e2e)** — `packages/mcp-server/src/index.ts` — stdio MCP server exposing `start_run`/`run_status`/`run_result`/`stop_run` (detached cele2e runs read off the event bus, no ANSI scraping) + `env_check` (docker VM, run-lock, shared-infra, mgmt image CLI version, netapps). Dev/ops tool, `private`, not shipped to consumers.

package/drizzle/0019_backup_pid.sql

@@ -0,0 +1,18 @@
+-- Record which process owns a backup's staging directory.
+--
+-- `backup-create.ts` assembles every envelope under `/tmp/celilo-backup-<id>`
+-- and removes it in a `finally`. A `finally` does not run when the process is
+-- killed by a signal — a dispatcher timeout, an OOM, a reboot — and those are
+-- exactly the cases that strand the largest directories. On celilo-mgr the
+-- scheduled sweep was killed at 60s against a backup needing ~5.5 minutes,
+-- three times an hour, stranding 27 GB in under six hours.
+--
+-- Reclamation therefore cannot depend on how a backup ends. The staging reaper
+-- (services/backup-staging.ts) asks "is anyone still using this directory?"
+-- instead, and this column is what lets it answer: the directory names its
+-- record, and the record names its process.
+--
+-- Nullable on purpose. Rows written before this column exists have no pid; the
+-- reaper keeps their staging until the TTL expires rather than guessing.
+
+ALTER TABLE `backups` ADD `pid` integer;

package/drizzle/meta/_journal.json

@@ -134,6 +134,13 @@
       "when": 1783400000000,
       "tag": "0018_drop_alert_policy_snapshot",
       "breakpoints": true
+    },
+    {
+      "idx": 19,
+      "version": "6",
+      "when": 1783500000000,
+      "tag": "0019_backup_pid",
+      "breakpoints": true
     }
   ]
 }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@celilo/cli",
-  "version": "0.16.2",
+  "version": "0.18.0",
   "description": "Celilo — home lab orchestration CLI",
   "type": "module",
   "bin": {
@@ -57,10 +57,10 @@
   },
   "dependencies": {
     "@aws-sdk/client-s3": "^3.1024.0",
-    "@celilo/capabilities": "^0.9.1",
-    "@celilo/cli-display": "^0.1.9",
-    "@celilo/core": "^0.3.0",
-    "@celilo/event-bus": "^0.1.10",
+    "@celilo/capabilities": "^0.10.0",
+    "@celilo/cli-display": "^0.1.10",
+    "@celilo/core": "^0.3.2",
+    "@celilo/event-bus": "^0.2.0",
     "@clack/prompts": "^1.1.0",
     "ajv": "^8.18.0",
     "drizzle-orm": "^0.36.4",

package/schemas/system_config.json

@@ -74,7 +74,7 @@
       "maximum": 4094,
       "description": "VLAN tag for internal zone (not defaulted; internal is untagged)"
     },
-    "network.vpn.subnet": {
+    "network.control-plane-vpn.subnet": {
       "type": "string",
       "pattern": "^\\d{1,3}\\.\\d{1,3}\\.\\d{1,3}\\.\\d{1,3}/\\d{1,2}$",
       "description": "WireGuard VPN client subnet CIDR (remote-access tunnel; consumed by technitium's split-horizon → in-zone caddy view, #275, and by the firewall's trusted sources once the wireguard module registers it)"

package/src/cli/command-tree-parser.ts

@@ -163,7 +163,6 @@ export class CommandTreeParser {
    * Strip ANSI color codes from text
    */
   private stripAnsiCodes(text: string): string {
-    // biome-ignore lint/suspicious/noControlCharactersInRegex: ANSI escape codes require control characters
     return text.replace(/\x1B\[[0-9;]*[a-zA-Z]/g, '');
   }
 

package/src/cli/commands/alerts-poll.ts

@@ -17,6 +17,7 @@ import { systemConfig } from '../../db/schema';
 import { makeReceiver, pollInbound } from '../../services/alerting/inbound-poller';
 import { startNotificationResponder } from '../../services/alerting/notification-responder';
 import { listRoutes } from '../../services/alerting/people';
+import { readLastRead, writeLastRead } from '../../services/alerting/read-records';
 import { loadNotificationTransport } from '../../services/alerting/transport-loader';
 import type { CommandResult } from '../types';
 
@@ -52,6 +53,26 @@ function writeCursor(
   }
 }
 
+/**
+ * " (last read OK 3h ago)" or " (never read successfully)".
+ *
+ * Appended to a read failure because the failure alone does not say how bad it
+ * is. One failed poll is a blip; a transport that has not been readable since
+ * Tuesday is an outage nobody was told about, and those two produced identical
+ * output until this record existed.
+ */
+function sinceLastSuccess(db: ReturnType<typeof getDb>, transportModuleId: string): string {
+  const last = readLastRead(db, transportModuleId)?.lastSuccessAt;
+  if (!last) return ' (never read successfully)';
+  const ms = Date.now() - new Date(last).getTime();
+  const mins = Math.floor(ms / 60_000);
+  if (mins < 60) return ` (last read OK ${mins}m ago)`;
+  const hours = Math.floor(mins / 60);
+  return hours < 48
+    ? ` (last read OK ${hours}h ago)`
+    : ` (last read OK ${Math.floor(hours / 24)}d ago)`;
+}
+
 export async function handleAlertsPoll(
   flags: Record<string, string | boolean> = {},
 ): Promise<CommandResult> {
@@ -97,6 +118,7 @@ export async function handleAlertsPoll(
     receiveFrom: makeReceiver(db),
     readCursor: (t) => readCursor(db, t),
     writeCursor: (t, c) => writeCursor(db, t, c),
+    recordRead: (t, r) => writeLastRead(db, t, r),
     now: () => new Date(),
     transportFor: (route) => loadNotificationTransport(db, route.transportModuleId),
     answerInterview: (eventId, value) => responder.answer(eventId, value),
@@ -122,7 +144,10 @@ export async function handleAlertsPoll(
   // an operator has no way to tell which — the ambiguity that made the Signal
   // ack path take a week to diagnose.
   const detail = [
-    ...report.failures.map((f) => `${f.transportModuleId} COULD NOT BE READ: ${f.error}`),
+    ...report.failures.map(
+      (f) =>
+        `${f.transportModuleId} COULD NOT BE READ: ${f.error}${sinceLastSuccess(db, f.transportModuleId)}`,
+    ),
     ...report.unheard.map((u) => `not heard from ${u.senderAddress}: ${u.reason}`),
     ...askErrors,
   ];

package/src/cli/commands/backup-sweep.ts

@@ -9,17 +9,68 @@
  * report counts.
  */
 
+import { readdirSync, rmSync } from 'node:fs';
+import { tmpdir } from 'node:os';
+import { join } from 'node:path';
 import {
   createModuleBackup,
   findBackupEligibleModules,
   isBackupDue,
 } from '../../services/backup-create';
+import { failBackup, getBackup } from '../../services/backup-metadata';
 import { pruneBackupsForModule } from '../../services/backup-retention';
+import {
+  ABANDONED_BACKUP_MESSAGE,
+  STAGING_PREFIX,
+  impliesAbandonedRecord,
+  reapOrphanedStaging,
+} from '../../services/backup-staging';
 import { type BackupSweepReport, runBackupSweep } from '../../services/backup-sweep';
+import { isPidRunnable } from '../../services/module-operations';
 import type { CommandResult } from '../types';
 
+/**
+ * Reclaim orphaned staging, then correct the records that were still claiming
+ * to be running.
+ *
+ * The record fix-up lives here rather than inside the reaper because the reaper
+ * is the filesystem decision and this is a database write; both follow from the
+ * one liveness lookup, so neither repeats it.
+ */
+function reapStaging() {
+  const report = reapOrphanedStaging({
+    listStagingDirs: () => {
+      // A missing or unreadable temp dir is not a reason to fail the sweep.
+      try {
+        return readdirSync(tmpdir())
+          .filter((name) => name.startsWith(STAGING_PREFIX))
+          .map((name) => join(tmpdir(), name));
+      } catch {
+        return [];
+      }
+    },
+    lookupOwner: (recordId) => {
+      const record = getBackup(recordId);
+      if (!record) return null;
+      return { status: record.status, pid: record.pid, startedAt: record.startedAt };
+    },
+    isPidRunnable,
+    remove: (path) => rmSync(path, { recursive: true, force: true }),
+    now: () => Date.now(),
+  });
+
+  for (const reclaimed of report.reclaimed) {
+    if (impliesAbandonedRecord(reclaimed.reason)) {
+      failBackup(reclaimed.recordId, ABANDONED_BACKUP_MESSAGE);
+    }
+  }
+
+  return report;
+}
+
 export async function handleBackupSweep(): Promise<CommandResult> {
   const report = await runBackupSweep({
+    reapStaging,
     listEligible: () =>
       findBackupEligibleModules().map(({ module, manifest }) => ({ id: module.id, manifest })),
     isDue: (moduleId, schedule) => isBackupDue(moduleId, schedule),
@@ -55,6 +106,17 @@ function formatReport(report: BackupSweepReport): string {
   if (report.failures.length > 0) parts.push(`${report.failures.length} FAILED`);
 
   const lines = [`backup sweep: ${parts.join(', ')}`];
+  // Reclamation is reported even though it is housekeeping: it is the only
+  // visible evidence that backups have been dying, and a silent reaper would
+  // hide the very failure it exists to clean up after.
+  if (report.staging.reclaimed.length > 0) {
+    lines.push(
+      `  reclaimed ${report.staging.reclaimed.length} orphaned staging dir(s) from backups that were killed before cleanup`,
+    );
+    for (const reclaimed of report.staging.reclaimed) {
+      lines.push(`    ${reclaimed.recordId} (${reclaimed.reason})`);
+    }
+  }
   for (const moduleId of report.skippedLocked) {
     lines.push(`  skipped ${moduleId}: another operation is in flight — retrying next tick`);
   }

package/src/cli/commands/module-operations.test.ts

@@ -6,7 +6,7 @@ import { eq } from 'drizzle-orm';
 import { closeDb, getDb } from '../../db/client';
 import { runMigrations } from '../../db/migrate';
 import { moduleOperations } from '../../db/schema';
-import { OPERATION_TTL_MS } from '../../services/module-operations';
+import { ABANDONED_RELEASE_MESSAGE, OPERATION_TTL_MS } from '../../services/module-operations';
 import { handleModuleOperations } from './module-operations';
 
 describe('celilo module operations', () => {
@@ -85,6 +85,50 @@ describe('celilo module operations', () => {
     expect(statusOf('expired')).toBe('in_progress');
   });
 
+  // The recurrence gate for #581: the sweep runs on every hourly tick, so an
+  // abandoned row must be reclaimed once and then stop being work. Before
+  // this, rows only ever accumulated — 85 of them, the oldest 62 days old.
+  it('reclaims an abandoned row and does not re-collect it on the next sweep', () => {
+    insert('expired', process.pid, OPERATION_TTL_MS + 60_000);
+
+    const first = handleModuleOperations(['clear'], {});
+    if (!first.success) throw new Error(`expected success, got: ${first.error}`);
+    expect(first.message).toContain('Released 1');
+    expect(statusOf('expired')).toBe('failed');
+
+    const second = handleModuleOperations(['clear'], {});
+    if (!second.success) throw new Error(`expected success, got: ${second.error}`);
+    expect(second.message).toContain('no operations in progress');
+
+    // Reclaimed, not deleted: the released row is the evidence the
+    // abandoned-operations audit reads.
+    const row = getDb()
+      .select()
+      .from(moduleOperations)
+      .where(eq(moduleOperations.id, 'expired'))
+      .get();
+    expect(row?.errorMessage).toBe(ABANDONED_RELEASE_MESSAGE);
+  });
+
+  it('hides abandoned rows from list by default and shows them with --abandoned', () => {
+    insert('expired', process.pid, OPERATION_TTL_MS + 60_000);
+
+    const lines: string[] = [];
+    const original = console.log;
+    console.log = (msg?: unknown) => lines.push(String(msg));
+    try {
+      handleModuleOperations(['list'], {});
+      expect(lines.join('\n')).not.toContain('pid');
+      expect(lines.join('\n')).toContain('1 abandoned row(s) hidden');
+
+      lines.length = 0;
+      handleModuleOperations(['list'], { abandoned: true });
+      expect(lines.join('\n')).toContain('abandoned (expired)');
+    } finally {
+      console.log = original;
+    }
+  });
+
   it('rejects an unknown action rather than silently listing', () => {
     const result = handleModuleOperations(['nuke'], {});
     if (result.success) throw new Error('expected an unknown action to fail');

package/src/cli/commands/module-operations.ts

@@ -10,12 +10,22 @@
  *
  * `clear` marks rows failed rather than deleting them — the history of
  * what was abandoned, and when, is worth more than a tidy table.
+ *
+ * That is load-bearing now, not merely tidy: `clear` runs hourly off the
+ * bus, and `services/audit/abandoned-operations.ts` reads exactly these
+ * released rows to notice that one module's backup is being killed over
+ * and over. Turning this into a DELETE would tidy the table and silently
+ * destroy the only signal that a repeatedly-dying operation ever leaves.
  */
 
 import { and, eq } from 'drizzle-orm';
 import { getDb } from '../../db/client';
 import { type ModuleOperation, moduleOperations } from '../../db/schema';
-import { OPERATION_TTL_MS, isPidRunnable } from '../../services/module-operations';
+import {
+  ABANDONED_RELEASE_MESSAGE,
+  OPERATION_TTL_MS,
+  isPidRunnable,
+} from '../../services/module-operations';
 import type { CommandResult } from '../types';
 
 /** Why a row is not holding the lock, or null when it still is. */
@@ -41,36 +51,49 @@ function inProgressRows(): ModuleOperation[] {
     .all();
 }
 
-function handleList(): CommandResult {
+/**
+ * Abandoned rows are summarised, not listed, unless `--abandoned` asks for
+ * them. The question this command answers is "what holds the lock right
+ * now", and on a fleet where something is dying repeatedly the answer was
+ * buried under 85 corpses. The count still prints, so they never become
+ * invisible — the audit is what reads them as a symptom
+ * (`services/audit/abandoned-operations.ts`).
+ */
+function handleList(flags: Record<string, boolean | string>): CommandResult {
   const now = Date.now();
   const rows = inProgressRows();
+  const showAbandoned = flags.abandoned === true || flags.all === true;
 
   if (rows.length === 0) {
     console.log('\nNo module operations in progress.\n');
     return { success: true, message: 'no operations in progress' };
   }
 
+  const holding = rows.filter((row) => abandonedReason(row, now) === null);
+  const abandoned = rows.length - holding.length;
+  const shown = showAbandoned ? rows : holding;
+
   console.log('\nModule operations in progress:\n');
-  let holding = 0;
-  for (const row of rows) {
+  for (const row of shown) {
     const reason = abandonedReason(row, now);
-    if (!reason) holding++;
     const age = formatAge(now - row.startedAt.getTime());
     const status = reason ? `abandoned (${reason})` : 'HOLDING LOCK';
     console.log(
       `  ${row.operation.padEnd(9)} ${row.moduleId.padEnd(16)} pid ${String(row.pid).padEnd(8)} ${age.padStart(4)} ago   ${status}`,
     );
   }
+  if (shown.length === 0) console.log('  (nothing is holding the lock)');
 
-  const abandoned = rows.length - holding;
   console.log('');
-  if (abandoned > 0) {
-    console.log(`${abandoned} abandoned row(s) — "celilo module operations clear" sweeps them.\n`);
+  if (abandoned > 0 && !showAbandoned) {
+    console.log(
+      `${abandoned} abandoned row(s) hidden — "--abandoned" lists them, the hourly sweep clears them.\n`,
+    );
   }
 
   return {
     success: true,
-    message: `${rows.length} in progress (${holding} holding the lock, ${abandoned} abandoned)`,
+    message: `${rows.length} in progress (${holding.length} holding the lock, ${abandoned} abandoned)`,
   };
 }
 
@@ -108,7 +131,7 @@ function handleClear(flags: Record<string, boolean | string>): CommandResult {
       .set({
         status: 'failed',
         completedAt: new Date(),
-        errorMessage: 'abandoned — released by "celilo module operations clear"',
+        errorMessage: ABANDONED_RELEASE_MESSAGE,
       })
       .where(and(eq(moduleOperations.id, row.id), eq(moduleOperations.status, 'in_progress')))
       .run();
@@ -124,11 +147,11 @@ export function handleModuleOperations(
 ): CommandResult {
   const action = args[0];
 
-  if (!action || action === 'list') return handleList();
+  if (!action || action === 'list') return handleList(flags);
   if (action === 'clear') return handleClear(flags);
 
   return {
     success: false,
-    error: `Unknown action "${action}"\n\nUsage: celilo module operations [list|clear] [--all]`,
+    error: `Unknown action "${action}"\n\nUsage: celilo module operations [list|clear] [--abandoned] [--all]`,
   };
 }

package/src/cli/commands/module-show.ts

@@ -171,6 +171,7 @@ export async function handleModuleShowZone(args: string[]): Promise<CommandResul
       secure: 'Secure (Authentication/Database)',
       'secure-mgmt': "Secure-Mgmt (celilo's own control plane)",
       external: 'External (VPS/Cloud)',
+      'control-plane-vpn': 'Control-plane VPN (administrative remote access)',
     };
 
     // Cast at the lookup, not the declaration: `zone` comes from config and may be