@celilo/cli 0.17.0 → 0.19.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (72) hide show
  1. package/CELILO_CORE_MODULES.md +1 -1
  2. package/CELILO_SUBSYSTEMS.md +38 -9
  3. package/drizzle/0019_backup_pid.sql +18 -0
  4. package/drizzle/meta/_journal.json +7 -0
  5. package/package.json +5 -5
  6. package/schemas/system_config.json +1 -1
  7. package/src/api/remote-client.test.ts +62 -0
  8. package/src/api/serve.ts +14 -6
  9. package/src/cli/command-tree-parser.ts +0 -1
  10. package/src/cli/commands/apt-upgrade.test.ts +20 -1
  11. package/src/cli/commands/apt-upgrade.ts +12 -2
  12. package/src/cli/commands/backup-sweep.ts +62 -0
  13. package/src/cli/commands/events.ts +90 -0
  14. package/src/cli/commands/module-operations.test.ts +45 -1
  15. package/src/cli/commands/module-operations.ts +35 -12
  16. package/src/cli/commands/module-show.ts +1 -0
  17. package/src/cli/commands/module-update.test.ts +72 -1
  18. package/src/cli/commands/module-update.ts +45 -22
  19. package/src/cli/commands/system-audit.ts +2 -0
  20. package/src/cli/commands/system-migrate.test.ts +56 -0
  21. package/src/cli/commands/system-migrate.ts +92 -4
  22. package/src/cli/commands/system-update.ts +5 -0
  23. package/src/cli/completion.ts +19 -0
  24. package/src/cli/fuel-gauge.ts +0 -1
  25. package/src/cli/generate-zsh-completion.ts +1 -1
  26. package/src/cli/index.ts +5 -1
  27. package/src/cli/tui/audit-state.ts +4 -0
  28. package/src/cli/tui/audit-tui.test.tsx +0 -1
  29. package/src/db/migration-status.test.ts +114 -0
  30. package/src/db/migration-status.ts +78 -0
  31. package/src/db/schema-introspection.ts +8 -1
  32. package/src/db/schema.ts +53 -9
  33. package/src/hooks/capability-loader.ts +30 -1
  34. package/src/ipam/allocator.ts +13 -3
  35. package/src/services/alerting/builtin-monitors.test.ts +42 -0
  36. package/src/services/alerting/builtin-monitors.ts +2 -0
  37. package/src/services/alerting/builtin-source.ts +15 -0
  38. package/src/services/audit/abandoned-operations.test.ts +73 -0
  39. package/src/services/audit/abandoned-operations.ts +0 -0
  40. package/src/services/audit/disk-space.test.ts +111 -0
  41. package/src/services/audit/disk-space.ts +114 -0
  42. package/src/services/audit/index.test.ts +1 -0
  43. package/src/services/audit/index.ts +9 -0
  44. package/src/services/audit/types.ts +2 -0
  45. package/src/services/backup-create.ts +4 -4
  46. package/src/services/backup-in-flight-refusal.test.ts +2 -0
  47. package/src/services/backup-metadata.ts +4 -0
  48. package/src/services/backup-staging.test.ts +134 -0
  49. package/src/services/backup-staging.ts +192 -0
  50. package/src/services/backup-sweep.test.ts +68 -0
  51. package/src/services/backup-sweep.ts +62 -0
  52. package/src/services/bus-interview.ts +11 -5
  53. package/src/services/config-interview.ts +1 -1
  54. package/src/services/deploy-ansible.ts +0 -1
  55. package/src/services/disk-probe.test.ts +74 -0
  56. package/src/services/disk-probe.ts +145 -0
  57. package/src/services/events-daemon.test.ts +244 -0
  58. package/src/services/events-daemon.ts +295 -8
  59. package/src/services/fleet-checks.test.ts +75 -4
  60. package/src/services/fleet-checks.ts +97 -12
  61. package/src/services/interview-errors.ts +20 -0
  62. package/src/services/module-operations.test.ts +22 -0
  63. package/src/services/module-operations.ts +48 -1
  64. package/src/services/module-subscriptions.test.ts +39 -6
  65. package/src/services/module-subscriptions.ts +6 -4
  66. package/src/services/module-types-generator.test.ts +6 -3
  67. package/src/services/module-types-generator.ts +12 -7
  68. package/src/services/remote-responder.test.ts +70 -0
  69. package/src/services/remote-responder.ts +27 -10
  70. package/src/services/responder-probe.ts +3 -1
  71. package/src/services/update/orchestrator.test.ts +1 -0
  72. package/src/variables/context.ts +6 -1
@@ -26,7 +26,7 @@ Each entry: `module id` — what it is — **provides** / **requires** capabilit
26
26
  - **knot-unbound-internal** — split-horizon internal DNS via Knot (authoritative) + Unbound (recursive); lightweight, plain apt, no .NET. **provides:** `dns_internal`. Ships a base-module-aspect (`modules/knot-unbound-internal/base-module-aspect/`).
27
27
  - **technitium** — internal split-horizon DNS resolver + authoritative server (web UI + HTTP API); heavier alternative to knot-unbound. **provides:** `dns_internal`. Ships a base-module-aspect (`modules/technitium/base-module-aspect/`).
28
28
  - **namecheap** — public DNS A-record management via Namecheap Dynamic DNS API (HTTP, no browser automation). **provides:** `dns_registrar`.
29
- - **wireguard** — owns the admin WireGuard tunnel on the firewall host: interface, listen port, peers (as records), and client subnet are module config rather than hand-maintained state. Exposes the listen port and **registers the client subnet as a trusted source**, so VPN reach into the managed zones is in the firewall registry and every converge re-emits it. Records `network.vpn.subnet`, which the internal resolver's split-horizon view also consumes. Adopts a running tunnel in place (existing key and peers retained; `wg syncconf`, never `wg-quick down`) because that tunnel is the operator's recovery path. **requires:** `firewall` (and the provider must support trusted-source registration — `iptables` does, `greenwave` does not).
29
+ - **wireguard** — owns the admin WireGuard tunnel on the firewall host: interface, listen port, peers (as records), and client subnet are module config rather than hand-maintained state. Exposes the listen port and **registers the client subnet as a trusted source**, so VPN reach into the managed zones is in the firewall registry and every converge re-emits it. Records `network.control-plane-vpn.subnet`, which the internal resolver's split-horizon view also consumes. Adopts a running tunnel in place (existing key and peers retained; `wg syncconf`, never `wg-quick down`) because that tunnel is the operator's recovery path. **requires:** `firewall` (and the provider must support trusted-source registration — `iptables` does, `greenwave` does not).
30
30
 
31
31
  ## Public edge (ingress / identity)
32
32
 
@@ -92,7 +92,7 @@ Module hooks reach a remote box ONLY through these typed primitives
92
92
  - **`runAppCommand` / `runAppCommandWithSecret`** — escape-hatch on-box command; the secret variant feeds the secret on **stdin** (`$SECRET`), never argv.
93
93
  - **`streamBackup` / `streamRestore` / `fetchFile` / `pushFile`** — binary-safe streaming via local shell redirect/pipe.
94
94
  - **`waitFor`** — predicate-poll combinator.
95
- - **`applyRenderedConfig`** — converge: write rendered config → validate → apply → rollback (one round-trip). Used by caddy (Caddyfile), knot (views), iptables (ruleset).
95
+ - **`applyRenderedConfig`** — converge: write rendered config → validate → apply → rollback (one round-trip). Used by caddy (Caddyfile), knot (views), iptables (ruleset). On success it RETAINS the file it replaced as `<path>.celilo-prev` — the durable "what celilo last rendered" record a whole-file converge diffs against.
96
96
  - **`installAuthorizedKey`** — ONE-TIME credential bootstrap for an off-fleet account: `ssh-copy-id` under a password taken from the child ENV (`SSH_ASKPASS_REQUIRE=force`, so no `sshpass` dependency and the password never lands in a command string). Idempotent; used by `external_web` onboarding so every later publish is key-based.
97
97
  - **`tailLog` / `grepLog`** — journald reads (regex / ignoreCase).
98
98
 
@@ -112,10 +112,34 @@ runner seam (`execRunner` real / `createMockRunner` for tests) lives in
112
112
  (`renderRuleset`): registry + firewall state → a complete `iptables-restore`
113
113
  file. Default-DROP FORWARD + established/related + egress + the coarse
114
114
  zone-tier matrix (dmz→app, app→secure from `network.<zone>.subnet`) + per-service DNAT allows.
115
+ Egress permission and egress TRANSLATION are emitted together or neither: a
116
+ blanket `-o <wan>` MASQUERADE on a leaf, one `-s <subnet> -o <uplink>` rule per
117
+ network on a DOWNSTREAM firewall (no external edge). An upstream does not
118
+ translate on a downstream firewall's behalf.
119
+ `subnetsNeedingTranslation(state)` is that set — EVERY network behind the
120
+ firewall (tiers ∪ every declared zone subnet ∪ control plane ∪ registered
121
+ trusted sources) minus whichever of them CONTAINS the egress interface's own
122
+ address (`egressIp`, read off the box). Not just the data-plane tiers: a
123
+ downstream firewall's egress leg IS its LAN leg, so an untranslated network is
124
+ unreachable from a LAN host as well as from the internet.
115
125
  - **Converge** — `modules/iptables/scripts/firewall-functions.ts` (`converge`):
116
- `exposeService`/`unexposeService` register into the store, then render + apply
117
- atomically via `applyRenderedConfig` (`iptables-restore`). Replaces the old
118
- per-rule `iptables -A`; the registry (not `iptables -L`) is the source of truth.
126
+ `exposeService`/`unexposeService` register into the store, then render, CHECK,
127
+ DIFF, and only then apply atomically via `applyRenderedConfig`
128
+ (`iptables-restore`). Replaces the old per-rule `iptables -A`; the registry
129
+ (not `iptables -L`) is the source of truth. It does NOT apply unconditionally:
130
+ a failing check or an unexplainable removal throws, leaving the working
131
+ ruleset in place (`config.force` overrides, on the record).
132
+ - **Render-time completeness checks (pure)** — `modules/iptables/scripts/ruleset-checks.ts`
133
+ (`checkRenderedSet`, `blockingFindings`, `formatConvergeRefusal`): the SEMANTIC
134
+ counterpart to `iptables-restore --test`, which only validates syntax. Reports
135
+ a zone permitted to egress with no translation covering it, a control plane
136
+ that cannot reach a zone it manages, and an unknown control plane (a warning,
137
+ distinct from "known and absent"). Runs under `config.dryRun` too.
138
+ - **Pre-apply ruleset diff (pure)** — `modules/iptables/scripts/ruleset-diff.ts`
139
+ (`parseRuleset`, `diffRuleset`, `driftAgainstLive`, `snapshotCommand`,
140
+ `parseSnapshot`): what the converge would remove, and whether a registry change
141
+ explains it. Exact set algebra against the retained previous render, not a
142
+ heuristic. Live-vs-persisted drift is reported, never decided on.
119
143
  - **Trusted-source registry (desired state)** — `trusted_sources` DB table
120
144
  (migration `0016`) + `apps/celilo/src/services/trusted-sources.ts`
121
145
  (`buildTrustedSourceStore`, `composeTrustedSubnets`). The sibling primitive to
@@ -144,7 +168,7 @@ runner seam (`execRunner` real / `createMockRunner` for tests) lives in
144
168
  - **Deploy pipeline** — `apps/celilo/src/services/module-deploy.ts`.
145
169
  - **DNS provider backfill** (re-emit registrations when a provider deploys) — `apps/celilo/src/services/dns-provider-backfill.ts` — `isDnsInternalProvider`, `backfillProviderDns`.
146
170
  - **Base-module aspects (fan-out across the fleet)** — `apps/celilo/src/services/aspect-runner.ts` — `planAspectFanOut`, `runAspectFanOut`, `maybeRunAspectForTrigger`. Aspect content lives in `modules/<m>/base-module-aspect/` (e.g. knot-unbound-internal, technitium).
147
- - **In-flight operation lock** — `apps/celilo/src/services/module-operations.ts` — `startOperation`/`completeOperation`/`failOperation` record deploy/uninstall/backup/restore in `module_operations`; `refuseIfInFlight`/`checkInFlight` are what backup and restore consult. Deploy and uninstall REGISTER but never check: it is a one-way guard protecting backup/restore consistency, not a general mutex (`openspec/specs/management-server-backup/spec.md` "In-flight operation refusal"). A row stops holding the lock once it is GONE, STOPPED/zombie (`isPidRunnable`, `ps -o state=` — `kill(pid,0)` calls a Ctrl-Z'd process alive), or older than `OPERATION_TTL_MS` (2h). The TTL is not redundancy: a pid is a recycled number, and once the pid space wraps an old row names an unrelated healthy process. Operator surface: `celilo module operations [list|clear] [--all]` (`apps/celilo/src/cli/commands/module-operations.ts`).
171
+ - **In-flight operation lock** — `apps/celilo/src/services/module-operations.ts` — `startOperation`/`completeOperation`/`failOperation` record deploy/uninstall/backup/restore in `module_operations`; `refuseIfInFlight`/`checkInFlight` are what backup and restore consult. Deploy and uninstall REGISTER but never check: it is a one-way guard protecting backup/restore consistency, not a general mutex (`openspec/specs/management-server-backup/spec.md` "In-flight operation refusal"). A row stops holding the lock once it is GONE, STOPPED/zombie (`isPidRunnable`, `ps -o state=` — `kill(pid,0)` calls a Ctrl-Z'd process alive), or older than `OPERATION_TTL_MS` (2h). The TTL is not redundancy: a pid is a recycled number, and once the pid space wraps an old row names an unrelated healthy process. Operator surface: `celilo module operations [list|clear] [--abandoned] [--all]` (`apps/celilo/src/cli/commands/module-operations.ts`); `list` shows only what holds the lock, abandoned rows are summarised unless `--abandoned`. Abandoned rows are reclaimed hourly by the `celilo-operations-sweep` bus subscriber (`timer.tick.1h` → `celilo module operations clear`, armed by `ensureOperationsSweepSubscriber` from module registration and `celilo system migrate`). `clear` MARKS rows failed rather than deleting them, and that is load-bearing: the `abandoned_operations` audit reads exactly those released rows to notice one module's operation dying over and over.
148
172
  - **Backup artifact encryption** — `apps/celilo/src/services/backup-cipher.ts` — `encryptFileToFile`/`decryptFileToFile`, file-in/file-out and streamed, used by every backup writer (`backup-create.ts`) and reader (`backup-restore.ts`, `restore-from-file.ts`). Do NOT route artifacts through `secrets/encryption.ts`: that API is string-in/string-out for short DB values, and feeding it a tar cost base64 (1.33x) then hex (2x) then `JSON.stringify` — ~9x the artifact in memory, which OOM-killed forgejo's 774 MB backup, and a hard ~805 MB ceiling from the max string length that no amount of RAM raises. On-disk format is `magic "CELILOBK" (8) | version (1) | iv (16) | ciphertext | GCM tag (16)`; the tag is a trailer because it does not exist until the last byte is encrypted. `decryptFileToFile` still reads the pre-2026-07 JSON-envelope artifacts, discriminating on the magic bytes — the envelope's own `schemaVersion` cannot serve, as it lives inside the encrypted tar.
149
173
 
150
174
  ## Generation & templating
@@ -174,7 +198,8 @@ is currently wrong, and routes carry the message to a person's phone. Design:
174
198
  - **Alert identity** — `apps/celilo/src/services/alerting/keys.ts` — the key grammar (`module:<id>[/check:<name>]`, `builtin:<check>[/<kind>:<target>]`) that makes "the same problem" the same alert across runs. `moduleAlertKey`, `moduleCheckAlertKey`, `builtinAlertKey`, `parseAlertKey`.
175
199
  - **Reconciliation** — `apps/celilo/src/services/alerting/reconcile.ts` (`reconcile`) — a successful run's failing-key set is authoritative and resolution is by SET DIFFERENCE (absent ⇒ resolved). A run whose outcome is `error` resolves NOTHING and fires a module-level alert instead: the false-all-clear guard.
176
200
  - **Monitor execution** — `apps/celilo/src/services/alerting/run-monitor.ts` (`runOneMonitor`) + `sweep.ts` (`selectDueMonitors`) + `builtin-monitors.ts` / `health-coverage.ts` (the built-in checks and the "module with no health check" coverage check).
177
- - **Scheduled audit categories (`builtin_check` monitors)** — `apps/celilo/src/services/alerting/builtin-source.ts` — `SCHEDULABLE_BUILTIN_CHECKS` is the list of `celilo system audit` categories cheap enough to run every sweep: `machines_reachable` and `backups`. Everything else in the audit needs the whole world injected (proxmox, terraform, registry) and is not schedulable. Enable one with `celilo monitor add backups --interval 1h`. The `backups` roster comes from `apps/celilo/src/services/audit/backup-source.ts` (`loadBackupAuditInfo`), shared with `celilo system audit` so both judge the same fleet.
201
+ - **Scheduled audit categories (`builtin_check` monitors)** — `apps/celilo/src/services/alerting/builtin-source.ts` — `SCHEDULABLE_BUILTIN_CHECKS` is the list of `celilo system audit` categories cheap enough to run every sweep: `machines_reachable`, `backups`, `disk_space`, and `abandoned_operations` (`apps/celilo/src/services/audit/abandoned-operations.ts` — ≥3 abandonments of the same (module, operation) in 7d, the fingerprint of an operation being killed mid-flight). Everything else in the audit needs the whole world injected (proxmox, terraform, registry) and is not schedulable. Enable one with `celilo monitor add backups --interval 1h`. Targets are tab-completable `completion.ts` reads `SCHEDULABLE_BUILTIN_CHECKS` directly rather than a hand-copied list, so a newly-schedulable check is completable immediately.
202
+ - **Disk-space check** — `apps/celilo/src/services/audit/disk-space.ts` (`auditDiskSpace`, pure over measurements) + `apps/celilo/src/services/disk-probe.ts` (`probeDiskUsage`). Thresholds: `drift` at 85%, `blocked` at 95% — early enough to act on, since a check that fires at exhaustion reports an outage rather than preventing one. ⚠️ **The local management box is MEASURED, not exempted.** `probeMachines()` deliberately reports the local box reachable without probing it (celilo has no SSH key for itself, and the question is meaningless there); copying that shortcut into a disk check would skip the host most likely to fill — the one that stages backups, caches modules and writes the logs, and the one that DID fill. Local reads `statfs`; remote runs `df -P /` over the same bounded SSH. `percentUsed` matches `df`'s capacity semantics (excludes root-reserved blocks) so an alert and an operator's own `df` agree. An unmeasurable host yields a `todo` finding — recorded, never paged, because `machines_reachable` is already paging for that host. Findings are subjected on the **hostname**, not the machine UUID, because suppression resolves a machine's ancestor key from the hostname (see #596, where `machines_reachable` gets this wrong and its alerts therefore never suppress anything). The `backups` roster comes from `apps/celilo/src/services/audit/backup-source.ts` (`loadBackupAuditInfo`), shared with `celilo system audit` so both judge the same fleet.
178
203
  - **The sweep** — `apps/celilo/src/services/alerting/sweep-runner.ts` (`runSweep`) — the ordered pass that makes alerting run by itself: run due monitors → promote past-grace alerts → re-evaluate suppression → flush quiet-hours deferrals → notify. Driven by `celilo alerts sweep` on `timer.tick.5m`. Never throws for one bad monitor.
179
204
  - **Suppression (topology-derived, never configured)** — `apps/celilo/src/services/alerting/suppression.ts` — `ancestorKeysFor`/`findSuppressor`/`machineAlertKey`. A firing machine explains its modules' failures; a firing capability provider explains its zone's consumers. Derived from `module_systems`, so it cannot drift from reality. Deploy windows: `deploy-hooks.ts` (`openDeployWindow`/`closeDeployWindows` — closed by module, so a crashed deploy self-heals).
180
205
  - **Escalation & quiet hours** — `escalation.ts` (`decideEscalation`, every reason to stay silent enumerated) + `quiet-hours.ts` (`isWithinQuietHours`, Intl-based and DST-safe). Quiet hours defer the MESSAGE while the escalation clock keeps running.
@@ -194,7 +219,9 @@ Creation, scheduling and freshness. A module declares an `on_backup` hook and a
194
219
 
195
220
  - **Creation** — `apps/celilo/src/services/backup-create.ts` — `createModuleBackup` (invokes the module's `on_backup` hook into an encrypted envelope), `createSystemStateBackup` (celilo.db), `findBackupEligibleModules`, `isBackupDue`. Storage destinations: `backup-storage.ts`. Retention: `backup-retention.ts` (`pruneBackupsForModule`). Restore: `backup-restore.ts`.
196
221
  - **Cadence (one accessor)** — `apps/celilo/src/services/backup-schedule.ts` — `effectiveBackupSchedule(manifest)`. An absent `backup.schedule` means `daily`, NOT `manual`; opting out takes an explicit `manual`. Both the freshness audit and the backup sweep must read cadence through this one function, or a module can be alerted-on but never backed up.
197
- - **The sweep** — `apps/celilo/src/services/backup-sweep.ts` — `runBackupSweep` (the pass that makes backups run by themselves: for each eligible module, is its declared cadence due → back it up → apply declared retention) + `ensureBackupSweepSubscriber`. Driven by `celilo backup sweep` on `timer.tick.1h` — the coarsest tick that can still serve an `hourly` cadence. Armed automatically: `registerModuleSubscriptions` registers the subscriber for any module declaring an `on_backup` hook, so it appears on install or `module update`. A run refused by the in-flight operation lock is a skip retried next tick, never a failure.
222
+ - **The sweep** — `apps/celilo/src/services/backup-sweep.ts` — `runBackupSweep` (the pass that makes backups run by themselves: reclaim orphaned staging → for each eligible module, is its declared cadence due → back it up → apply declared retention) + `ensureBackupSweepSubscriber`. Driven by `celilo backup sweep` on `timer.tick.1h` — the coarsest tick that can still serve an `hourly` cadence. Armed from BOTH `registerModuleSubscriptions` (any module declaring an `on_backup` hook, so it appears on install or `module update`) AND `celilo system migrate`, which the `.deb` postinst runs on every apt upgrade — registering only from module install/update meant a corrected budget never reached an existing fleet, since the row already exists and module updates can be weeks apart. A run refused by the in-flight operation lock is a skip retried next tick, never a failure.
223
+ - **⚠️ The sweep's budget is stated, never inherited** — `BACKUP_SWEEP_TIMEOUT_MS` (4h) and `BACKUP_SWEEP_MAX_ATTEMPTS` (1) in `backup-sweep.ts`. The event bus defaults to `timeout_ms: 60000` / `max_attempts: 3`, and inheriting them made scheduled backups structurally impossible: one forgejo backup measured ~5.5 minutes (1.3 GB result, 3.9 GB peak staging), so the dispatcher SIGTERMed it at 60s — three times an hour, for days. The retry count is half the bug, not a detail: an impossible pass retried 3x strands 3x the staging (27 GB in 5.7 hours on celilo-mgr). The hourly tick IS the retry.
224
+ - **Staging reclamation** — `apps/celilo/src/services/backup-staging.ts` — `reapOrphanedStaging`, `stagingDirFor` (the single source of truth for `/tmp/celilo-backup-<record.id>`, shared with `backup-create.ts` so writer and reaper cannot drift). `backup-create.ts` removes its staging in a `finally`, which is correct and NOT enough: a `finally` never runs when the process is killed by a signal — dispatcher timeout, OOM, Ctrl-C, reboot — and those strand the LARGEST directories. So reclamation is kill-mode agnostic by construction: it asks "is anyone still using this?", answered from the `backups.pid` column plus `isPidRunnable`, both of which outlive the process. A directory is removed only when its owner is provably gone (record absent, record terminal, pid dead, or past `STAGING_TTL_MS` = 6h — the TTL is the only check surviving pid reuse). A live backup is always kept; an unrecognised name is ignored, never deleted. Reclaiming a record that still claimed `in_progress` also marks it failed with `ABANDONED_BACKUP_MESSAGE`, so `celilo backup list` stops showing phantom in-flight backups and the `backups` drift check cannot read a dead attempt as a fresh backup.
198
225
  - **Freshness audit** — `apps/celilo/src/services/audit/backups.ts` (`auditBackups`) — `backup_missing` / `backup_stale` drift findings against the same declared cadence.
199
226
  - **CLI** — `apps/celilo/src/cli/commands/` — `backup-sweep.ts`, `backup-create.ts` (also `celilo module backup`), `backup-list.ts`, `backup-restore.ts`, `backup-prune.ts`, `backup-delete.ts`, `backup-import.ts`, `backup-pull.ts`, `backup-name.ts`.
200
227
  - **Storage destinations CLI** — `storage-add-local.ts`, `storage-add-s3.ts`, `storage-list.ts`, `storage-verify.ts`, `storage-set-default.ts`, `storage-set-path.ts` (relocate a local destination, migrating existing archives unless `--no-migrate`), `storage-remove.ts`. Any credential change goes through `updateStorageCredentials` in `backup-storage.ts`, which clears the verification stamp — a `✓ Verified` must never describe a destination it was not measured against (#566).
@@ -205,7 +232,9 @@ Creation, scheduling and freshness. A module declares an `on_backup` hook and a
205
232
 
206
233
  ## Events
207
234
 
208
- - **Event bus** — `packages/event-bus/src/index.ts` — `Bus`, `openBus`, `defineEvents`, `defineHandler`, `runDispatcher`, pattern matching + timer ticks (`emitDueTimerTicks`, `retentionSweep`).
235
+ - **Migration interrogation** — `apps/celilo/src/db/migration-status.ts` — `getMigrationStatus(sqlite, migrationsFolder)` reports applied count, latest applied migration BY TAG, and pending ones by name (joining `__drizzle_migrations.created_at` to drizzle journal `when`, which is exact). Surface: `celilo system migrate --status`, which opens the DB **read-only** on purpose — `getDb()` auto-migrates on open, so a status routed through it would repair what it claims to report and could never say "pending". Paired with `findSchemaDrift` (`db/schema-introspection.ts`), which is column-aware: a table COUNT cannot distinguish "the column migration applied" from "nothing happened", which is why a rollout asserting `backups.pid` had to reach for `sqlite3` over SSH. `checkSchemaDrift` (`services/fleet-checks.ts`) fails on a missing table, a missing column, OR an unapplied journal migration, and its summary names tables AND columns so the operator can see what was checked.
236
+ - **Event bus** — `packages/event-bus/src/index.ts` — `Bus`, `openBus`, `defineEvents`, `defineHandler`, `runDispatcher`, pattern matching + timer ticks (`emitDueTimerTicks`, `retentionSweep`). **Exactly one dispatcher per bus**: `runDispatcher` refuses to start while another dispatcher's process is alive (`assertSoleDispatcher` in `dispatcher.ts`, liveness via `bus.liveDispatchers()` — `kill(pid,0)`, not heartbeat age, since a tick blocks for as long as its slowest handler). The exclusion is here rather than in the systemd unit because a stranded dispatcher can sit outside the unit's cgroup where `KillMode` cannot reach it (#580). `bus.health()` reports `dispatcherCount`/`dispatchers` and a `duplicate_dispatcher` status; `checkDispatcher` (`services/fleet-checks.ts`) fails on more than one. **Supervision** (`services/events-daemon.ts`): the unit is named `celilo-events.service` in BOTH the user and system scope, so the two are indistinguishable in every operator-facing string — `install-daemon` (which defaults to **user** scope) therefore refuses when the other scope's unit exists, and `checkDispatcher` compares the live pid against each installed unit's `MainPID` (`unitMainPid`) rather than merely testing that a unit file exists, since a file nobody is running is not supervision (#610).
237
+ - **Dispatcher supervision (install / restart)** — `apps/celilo/src/services/events-daemon.ts` — `installDaemon`/`planDaemonInstall`/`uninstallDaemon`/`readInstalledUnit` write the systemd unit or launchd plist without touching supervisor state, and `restartDaemon` (+ pure `orphanDispatcherPids`, `resolveRestartScope`, `supervisorCommands`) is the one verb that DOES cycle it. CLI: `celilo events install-daemon|uninstall-daemon|show-daemon|restart-daemon [--system]`. `restart-daemon` exists because the dispatcher runs the code it LOADED: celilo-mgr sat 9 days on event-bus v0.1.8 after apt installed v0.2.0, running the very bug the release fixed (celilo#604). Two things make it non-trivial and both are load-bearing: (1) it stops any live dispatcher the supervisor does not own first — an ORPHAN (PPID 1) is invisible to `systemctl restart`, and `assertSoleDispatcher` then crash-loops the unit while the old code keeps serving; (2) it verifies on the BUS that a NEW pid is live reporting `BUS_VERSION`, never off systemctl's exit code, which returns 0 into exactly that crash loop. System scope shells `sudo systemctl` (the unit is root-owned; celilo runs unprivileged), covered by the scoped `/etc/sudoers.d/celilo-events-restart` conffile that `celilo-bootstrap` ships — a unit test asserts the argv and the grant cannot drift apart.
209
238
 
210
239
  ## Remote API (drive the CLI over the wire)
211
240
 
@@ -219,7 +248,7 @@ Run any celilo command on celilo-mgr over SSH instead of screen-scraping `ssh <h
219
248
  - **Access control** — `apps/celilo/src/services/api-access.ts` — `grantPrincipal`, `isAuthorized` (deny-by-default, `command:subcommand` grants), `renderAuthorizedKeys`. Table: `api_principals` (`apps/celilo/src/db/schema.ts`). CLI: `apps/celilo/src/cli/commands/api.ts` (`api grant|list|revoke|authorized-keys|key new`).
220
249
  - **Mid-run interview bridge (`kind:daemon` responder)** — `apps/celilo/src/services/remote-responder.ts` — `startRemoteResponder` bridges bus `interview.required.*` ↔ wire.
221
250
  - **Server provisioning** — the `celilo-bootstrap` deb (`packaging/celilo-bootstrap/scripts/postinst`) creates the non-root `celilo-api` landing account + sshd; membership in the `celilo` group + `/etc/sudoers.d/celilo` (`!use_pty`) gives api-serve DB access via the wrapper's sudo-drop.
222
- - **Self-upgrade (apt)** — `celilo apt-upgrade` (`apps/celilo/src/cli/commands/apt-upgrade.ts`) upgrades the deb-installed `celilo`/`celilo-bootstrap` packages (`apt-get update` → `--only-upgrade install`) then spawns a fresh `celilo system migrate` (ISS-0100). It's the RW target behind the MCP's registry-derived `celilo_apt_upgrade` tool; the celilo user's two apt invocations are scoped-sudo'd by `/etc/sudoers.d/celilo-apt-upgrade`, shipped by `celilo-bootstrap`. **This upgrades celilo ITSELF — not the modules it manages. For those, see Module auto-upgrade below; the two are routinely confused.**
251
+ - **Self-upgrade (apt)** — `celilo apt-upgrade` (`apps/celilo/src/cli/commands/apt-upgrade.ts`) upgrades the deb-installed `celilo`/`celilo-bootstrap` packages (`apt-get update` → `--only-upgrade install`) then spawns a fresh `celilo system migrate` (ISS-0100), then `celilo events restart-daemon` so the dispatcher actually runs the code just installed — a failure there fails the whole command and names which steps DID complete, because "upgraded" while the dispatcher serves stale code is the silent state celilo#604 documents. It's the RW target behind the MCP's registry-derived `celilo_apt_upgrade` tool; the celilo user's two apt invocations are scoped-sudo'd by `/etc/sudoers.d/celilo-apt-upgrade`, shipped by `celilo-bootstrap`. **This upgrades celilo ITSELF — not the modules it manages. For those, see Module auto-upgrade below; the two are routinely confused.**
223
252
  - **Module auto-upgrade (registry-poll CD)** — the *pull* half of continuous deployment: celilo-mgr polls the registry and upgrades opted-in modules unattended. Spec: `openspec/specs/module-auto-upgrade/spec.md`. Entry points: `apps/celilo/src/cli/commands/module-upgrade.ts` — `runRegistryPoll` (the `--poll` path), `selectPollTargets` (pure: `autoUpgrade && latest && change ∉ {up-to-date, ahead}`), `upgradeOneModule` (update → backup → deploy → verify), `needsPreUpgradeBackup`, `pickAutoUpgrade`/`pickUpgradePolicy` (both fail closed/safe); `classifyVersionChange` in `module-update.ts` (treats a registry `+N` revision as a patch); `resolveDeployPosture` in `apps/celilo/src/services/deploy-posture.ts`. Trigger: celilo-mgmt's `registry-poll` subscription (`modules/celilo-mgmt/manifest.yml`) on `timer.tick.15m` with handler **`celilo module upgrade --poll`** — the flag is REQUIRED, since the dispatcher appends the event id positionally and a bare handler would consume it as the optional module name (silent: 3108 deliveries, 0 successes). Operator controls are framework config keys settable on ANY module (`FRAMEWORK_CONFIG_KEYS` in `module-config.ts`): `auto_upgrade` (opt-in, default false) and `upgrade_policy` (`by-semver`|`always-safe`|`always-fast`), validated at set time because both readers fail open. ⚠️ `always-safe` guarantees safe *posture*, NOT a backup — `needsPreUpgradeBackup` also requires the TARGET manifest to declare an `on_backup` hook, else it warns and proceeds. Confirm a data-bearing module declares `on_backup` before enabling `auto_upgrade` on it. The *build* half (app CI publishing a `.netapp` on merge) is not yet shipped — `openspec/changes/build-bus-poll-cd`.
224
253
  - **MCP service (`@celilo/mcp`)** — `packages/mcp/src/` — an operator-facing stdio MCP server (official `@modelcontextprotocol/sdk`, bin `celilo-mcp`) that drives a remote celilo server over the Remote API for an AI client. Two-item config (`config.ts`: `server` + `defaultUser`, env or `~/.config/celilo-mcp/config.json`). Dual-principal auth (`auth.ts`: `celilo-mcp auth setup` enrolls read-only `celilo-mcp-ro` + full `celilo-mcp-rw` ed25519 keypairs, prints the exact `celilo api grant` lines the operator runs server-side). Transport (`transport.ts`): reuses `@celilo/core` `runRemoteClient`, selecting the principal by `ssh -i <key>` and capturing structured output. Tool surface is generated LIVE from the server's command registry — `registry-fetch.ts` fetches `celilo commands --json` (+ `service list --json` for configured providers) over the RO principal on connect; `tools-from-registry.ts` (pure) projects that into one tool per runnable leaf, grouped by top-level command (`celilo_module_*`, `celilo_proxmox_*`, …), each with a Zod input schema from the leaf's args/flags and a read/write tag → RO/RW routing, plus a generic `celilo_run` escape hatch. Auto-detect hides provider-gated groups (e.g. `celilo_proxmox_*` until a Proxmox service is configured) and re-detects on a timer, emitting `notifications/tools/list_changed` when the surface changes. Coverage gate (`tests/coverage.test.ts`) asserts every registry leaf maps to a tool. Composite RO troubleshooting tools (`troubleshoot.ts` pure correlation + `troubleshoot-tools.ts` thin adapters): `celilo_assess_module <id>` and `celilo_fleet_status` correlate `celilo audit --json` (the drift backbone) with the `module list --json` roster into a per-module / fleet-wide verdict. Design: `openspec/changes/celilo-mcp-service/proposal.md`. (Distinct from the dev/ops `@celilo/mcp-server` below.)
225
254
 
@@ -0,0 +1,18 @@
1
+ -- Record which process owns a backup's staging directory.
2
+ --
3
+ -- `backup-create.ts` assembles every envelope under `/tmp/celilo-backup-<id>`
4
+ -- and removes it in a `finally`. A `finally` does not run when the process is
5
+ -- killed by a signal — a dispatcher timeout, an OOM, a reboot — and those are
6
+ -- exactly the cases that strand the largest directories. On celilo-mgr the
7
+ -- scheduled sweep was killed at 60s against a backup needing ~5.5 minutes,
8
+ -- three times an hour, stranding 27 GB in under six hours.
9
+ --
10
+ -- Reclamation therefore cannot depend on how a backup ends. The staging reaper
11
+ -- (services/backup-staging.ts) asks "is anyone still using this directory?"
12
+ -- instead, and this column is what lets it answer: the directory names its
13
+ -- record, and the record names its process.
14
+ --
15
+ -- Nullable on purpose. Rows written before this column exists have no pid; the
16
+ -- reaper keeps their staging until the TTL expires rather than guessing.
17
+
18
+ ALTER TABLE `backups` ADD `pid` integer;
@@ -134,6 +134,13 @@
134
134
  "when": 1783400000000,
135
135
  "tag": "0018_drop_alert_policy_snapshot",
136
136
  "breakpoints": true
137
+ },
138
+ {
139
+ "idx": 19,
140
+ "version": "6",
141
+ "when": 1783500000000,
142
+ "tag": "0019_backup_pid",
143
+ "breakpoints": true
137
144
  }
138
145
  ]
139
146
  }
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@celilo/cli",
3
- "version": "0.17.0",
3
+ "version": "0.19.0",
4
4
  "description": "Celilo — home lab orchestration CLI",
5
5
  "type": "module",
6
6
  "bin": {
@@ -57,10 +57,10 @@
57
57
  },
58
58
  "dependencies": {
59
59
  "@aws-sdk/client-s3": "^3.1024.0",
60
- "@celilo/capabilities": "^0.9.1",
61
- "@celilo/cli-display": "^0.1.9",
62
- "@celilo/core": "^0.3.1",
63
- "@celilo/event-bus": "^0.1.10",
60
+ "@celilo/capabilities": "^0.10.0",
61
+ "@celilo/cli-display": "^0.1.10",
62
+ "@celilo/core": "^0.4.0",
63
+ "@celilo/event-bus": "^0.2.0",
64
64
  "@clack/prompts": "^1.1.0",
65
65
  "ajv": "^8.18.0",
66
66
  "drizzle-orm": "^0.36.4",
@@ -74,7 +74,7 @@
74
74
  "maximum": 4094,
75
75
  "description": "VLAN tag for internal zone (not defaulted; internal is untagged)"
76
76
  },
77
- "network.vpn.subnet": {
77
+ "network.control-plane-vpn.subnet": {
78
78
  "type": "string",
79
79
  "pattern": "^\\d{1,3}\\.\\d{1,3}\\.\\d{1,3}\\.\\d{1,3}/\\d{1,2}$",
80
80
  "description": "WireGuard VPN client subnet CIDR (remote-access tunnel; consumed by technitium's split-horizon → in-zone caddy view, #275, and by the firewall's trusted sources once the wireguard module registers it)"
@@ -89,3 +89,65 @@ test('renders a forwarded interview and sends the answer back', async () => {
89
89
  expect(answer).toBeDefined();
90
90
  expect(JSON.parse(answer as string)).toEqual({ type: 'answer', id: 'q1', value: 'myhost' });
91
91
  });
92
+
93
+ /**
94
+ * Regression for the fabricated "operator declined".
95
+ *
96
+ * The default renderer prompts with clack, which reads keypresses off stdin
97
+ * whether or not stdin is a terminal. Driven over the MCP (stdin = the JSON-RPC
98
+ * stream) the next newline submitted the prompt at its `initialValue` — the
99
+ * question's `defaultValue` — so a breaking update nobody saw came back as a
100
+ * considered "no". With no terminal the client must say it cannot answer.
101
+ */
102
+ test('no TTY and no renderer → answers with an error, never the default', async () => {
103
+ const writes: string[] = [];
104
+ const encoder = new TextEncoder();
105
+ let controller!: ReadableStreamDefaultController<Uint8Array>;
106
+ const stdout = new ReadableStream<Uint8Array>({
107
+ start(c) {
108
+ controller = c;
109
+ },
110
+ });
111
+ const push = (obj: unknown) => controller.enqueue(encoder.encode(`${JSON.stringify(obj)}\n`));
112
+
113
+ const transport: RemoteTransport = {
114
+ stdin: {
115
+ write(chunk: string) {
116
+ writes.push(chunk);
117
+ if (chunk.includes('"answer"')) {
118
+ push({ type: 'result', success: false, exitCode: 1 });
119
+ controller.close();
120
+ }
121
+ },
122
+ },
123
+ stdout,
124
+ kill() {},
125
+ exited: Promise.resolve(1),
126
+ };
127
+
128
+ push({ type: 'ready', protocolVersion: 1 });
129
+ push({
130
+ type: 'interview',
131
+ id: 'q1',
132
+ scope: 'module-upgrade:iptables',
133
+ key: 'apply_breaking',
134
+ kind: 'confirm',
135
+ message: 'Apply breaking update for iptables (1.0.2+9 → 2.0.0+1)?',
136
+ defaultValue: 'false',
137
+ });
138
+
139
+ // No `renderInterview` — exactly what runRemoteCapture used to do. bun test
140
+ // runs with a piped stdin, i.e. the MCP server's situation.
141
+ expect(process.stdin.isTTY).toBeFalsy();
142
+ const code = await runRemoteClient('ignored', ['module', 'update'], {
143
+ openTransport: () => transport,
144
+ out: { write() {} },
145
+ });
146
+
147
+ expect(code).toBe(1);
148
+ const answer = writes.find((w) => w.includes('"answer"'));
149
+ expect(answer).toBeDefined();
150
+ const parsed = JSON.parse(answer as string) as { value: unknown; error?: string };
151
+ expect(parsed.value).not.toBe(false);
152
+ expect(parsed.error).toContain('module-upgrade:iptables.apply_breaking');
153
+ });
package/src/api/serve.ts CHANGED
@@ -108,14 +108,19 @@ export async function apiServeMode(principal: string): Promise<void> {
108
108
  send({ type: 'ready', protocolVersion: API_PROTOCOL_VERSION });
109
109
 
110
110
  const busDbPath = getEventBusPath();
111
- const pendingAnswers = new Map<string, (value: unknown) => void>();
111
+ const pendingAnswers = new Map<
112
+ string,
113
+ { resolve: (value: unknown) => void; reject: (error: Error) => void }
114
+ >();
112
115
 
113
116
  const ask = (interview: WireInterview): Promise<unknown> =>
114
- new Promise((resolve) => {
115
- pendingAnswers.set(interview.id, resolve);
117
+ new Promise((resolve, reject) => {
118
+ pendingAnswers.set(interview.id, { resolve, reject });
116
119
  send({
117
120
  type: 'interview',
118
121
  id: interview.id,
122
+ scope: interview.scope,
123
+ key: interview.key,
119
124
  kind: interview.kind,
120
125
  message: interview.message,
121
126
  description: interview.description,
@@ -140,10 +145,13 @@ export async function apiServeMode(principal: string): Promise<void> {
140
145
  }
141
146
 
142
147
  if (msg.type === 'answer') {
143
- const resolve = pendingAnswers.get(msg.id);
144
- if (resolve) {
148
+ const pending = pendingAnswers.get(msg.id);
149
+ if (pending) {
145
150
  pendingAnswers.delete(msg.id);
146
- resolve(msg.value);
151
+ // `error` = the client couldn't reach a decider. Fail the waiting
152
+ // command rather than letting the question fall back to its default.
153
+ if (msg.error) pending.reject(new Error(msg.error));
154
+ else pending.resolve(msg.value);
147
155
  }
148
156
  continue;
149
157
  }
@@ -163,7 +163,6 @@ export class CommandTreeParser {
163
163
  * Strip ANSI color codes from text
164
164
  */
165
165
  private stripAnsiCodes(text: string): string {
166
- // biome-ignore lint/suspicious/noControlCharactersInRegex: ANSI escape codes require control characters
167
166
  return text.replace(/\x1B\[[0-9;]*[a-zA-Z]/g, '');
168
167
  }
169
168
 
@@ -2,7 +2,7 @@ import { describe, expect, test } from 'bun:test';
2
2
  import { handleAptUpgrade } from './apt-upgrade';
3
3
 
4
4
  describe('handleAptUpgrade', () => {
5
- test('runs the three steps in order when each succeeds', async () => {
5
+ test('runs every step in order when each succeeds, ending with the dispatcher restart', async () => {
6
6
  const seen: string[][] = [];
7
7
  const result = await handleAptUpgrade([], {}, (argv) => {
8
8
  seen.push(argv);
@@ -14,9 +14,28 @@ describe('handleAptUpgrade', () => {
14
14
  ['sudo', 'apt-get', 'update'],
15
15
  ['sudo', 'apt-get', '-y', '--only-upgrade', 'install', 'celilo', 'celilo-bootstrap'],
16
16
  ['/usr/local/bin/celilo', 'system', 'migrate'],
17
+ // celilo#604: without this, apt installs new code and the running
18
+ // dispatcher keeps serving the old — celilo-mgr did so for 9 days.
19
+ ['/usr/local/bin/celilo', 'events', 'restart-daemon'],
17
20
  ]);
18
21
  });
19
22
 
23
+ test('fails, and names what DID complete, when the dispatcher restart fails', async () => {
24
+ const result = await handleAptUpgrade([], {}, (argv) => ({
25
+ status: argv.includes('restart-daemon') ? 1 : 0,
26
+ }));
27
+
28
+ // The packages ARE upgraded and the dispatcher is NOT on the new code.
29
+ // Saying so in the failure is the acceptance condition — silence here is
30
+ // what let a stale dispatcher pass for a successful upgrade.
31
+ expect(result.success).toBe(false);
32
+ if (!result.success) {
33
+ expect(result.error).toContain('restart the event dispatcher');
34
+ expect(result.error).toContain('apt-get upgrade');
35
+ expect(result.error).toContain('apply DB migrations');
36
+ }
37
+ });
38
+
20
39
  test('stops at the first failing step and does not run later ones', async () => {
21
40
  const seen: string[][] = [];
22
41
  const result = await handleAptUpgrade([], {}, (argv) => {
@@ -34,6 +34,11 @@ const STEPS: Step[] = [
34
34
  argv: ['sudo', 'apt-get', '-y', '--only-upgrade', 'install', 'celilo', 'celilo-bootstrap'],
35
35
  },
36
36
  { label: 'apply DB migrations', argv: [CELILO_BIN, 'system', 'migrate'] },
37
+ // The dispatcher runs the code it LOADED, not the code on disk. Without this
38
+ // step celilo-mgr sat 9 days on event-bus v0.1.8 after apt installed v0.2.0,
39
+ // faithfully running the bug the upgrade shipped to fix (celilo#604). The
40
+ // upgraded binary does the restart so the verification is the new code's.
41
+ { label: 'restart the event dispatcher', argv: [CELILO_BIN, 'events', 'restart-daemon'] },
37
42
  ];
38
43
 
39
44
  /** Run one argv, inheriting stdio so its output streams through api-serve. */
@@ -46,18 +51,23 @@ export async function handleAptUpgrade(
46
51
  _flags: Record<string, string | boolean>,
47
52
  runStep: StepRunner = defaultRunner,
48
53
  ): Promise<CommandResult> {
54
+ const done: string[] = [];
49
55
  for (const step of STEPS) {
50
56
  process.stdout.write(`\n▸ ${step.label}\n`);
51
57
  const { status } = runStep(step.argv);
52
58
  if (status !== 0) {
59
+ // Name what DID happen. A failure on the last step means new code is
60
+ // installed and the dispatcher is still serving the old — the operator
61
+ // has to be told that in the failure itself, not left to infer it.
53
62
  return {
54
63
  success: false,
55
- error: `apt-upgrade failed at "${step.label}" (exit ${status ?? 'signal'}). Nothing further was run.`,
64
+ error: `apt-upgrade failed at "${step.label}" (exit ${status ?? 'signal'}). Completed: ${done.length > 0 ? done.join(', ') : 'nothing'}. Nothing further was run.`,
56
65
  };
57
66
  }
67
+ done.push(step.label);
58
68
  }
59
69
  return {
60
70
  success: true,
61
- message: 'celilo apt packages upgraded and migrations applied.',
71
+ message: 'celilo apt packages upgraded, migrations applied, dispatcher restarted on new code.',
62
72
  };
63
73
  }
@@ -9,17 +9,68 @@
9
9
  * report counts.
10
10
  */
11
11
 
12
+ import { readdirSync, rmSync } from 'node:fs';
13
+ import { tmpdir } from 'node:os';
14
+ import { join } from 'node:path';
12
15
  import {
13
16
  createModuleBackup,
14
17
  findBackupEligibleModules,
15
18
  isBackupDue,
16
19
  } from '../../services/backup-create';
20
+ import { failBackup, getBackup } from '../../services/backup-metadata';
17
21
  import { pruneBackupsForModule } from '../../services/backup-retention';
22
+ import {
23
+ ABANDONED_BACKUP_MESSAGE,
24
+ STAGING_PREFIX,
25
+ impliesAbandonedRecord,
26
+ reapOrphanedStaging,
27
+ } from '../../services/backup-staging';
18
28
  import { type BackupSweepReport, runBackupSweep } from '../../services/backup-sweep';
29
+ import { isPidRunnable } from '../../services/module-operations';
19
30
  import type { CommandResult } from '../types';
20
31
 
32
+ /**
33
+ * Reclaim orphaned staging, then correct the records that were still claiming
34
+ * to be running.
35
+ *
36
+ * The record fix-up lives here rather than inside the reaper because the reaper
37
+ * is the filesystem decision and this is a database write; both follow from the
38
+ * one liveness lookup, so neither repeats it.
39
+ */
40
+ function reapStaging() {
41
+ const report = reapOrphanedStaging({
42
+ listStagingDirs: () => {
43
+ // A missing or unreadable temp dir is not a reason to fail the sweep.
44
+ try {
45
+ return readdirSync(tmpdir())
46
+ .filter((name) => name.startsWith(STAGING_PREFIX))
47
+ .map((name) => join(tmpdir(), name));
48
+ } catch {
49
+ return [];
50
+ }
51
+ },
52
+ lookupOwner: (recordId) => {
53
+ const record = getBackup(recordId);
54
+ if (!record) return null;
55
+ return { status: record.status, pid: record.pid, startedAt: record.startedAt };
56
+ },
57
+ isPidRunnable,
58
+ remove: (path) => rmSync(path, { recursive: true, force: true }),
59
+ now: () => Date.now(),
60
+ });
61
+
62
+ for (const reclaimed of report.reclaimed) {
63
+ if (impliesAbandonedRecord(reclaimed.reason)) {
64
+ failBackup(reclaimed.recordId, ABANDONED_BACKUP_MESSAGE);
65
+ }
66
+ }
67
+
68
+ return report;
69
+ }
70
+
21
71
  export async function handleBackupSweep(): Promise<CommandResult> {
22
72
  const report = await runBackupSweep({
73
+ reapStaging,
23
74
  listEligible: () =>
24
75
  findBackupEligibleModules().map(({ module, manifest }) => ({ id: module.id, manifest })),
25
76
  isDue: (moduleId, schedule) => isBackupDue(moduleId, schedule),
@@ -55,6 +106,17 @@ function formatReport(report: BackupSweepReport): string {
55
106
  if (report.failures.length > 0) parts.push(`${report.failures.length} FAILED`);
56
107
 
57
108
  const lines = [`backup sweep: ${parts.join(', ')}`];
109
+ // Reclamation is reported even though it is housekeeping: it is the only
110
+ // visible evidence that backups have been dying, and a silent reaper would
111
+ // hide the very failure it exists to clean up after.
112
+ if (report.staging.reclaimed.length > 0) {
113
+ lines.push(
114
+ ` reclaimed ${report.staging.reclaimed.length} orphaned staging dir(s) from backups that were killed before cleanup`,
115
+ );
116
+ for (const reclaimed of report.staging.reclaimed) {
117
+ lines.push(` ${reclaimed.recordId} (${reclaimed.reason})`);
118
+ }
119
+ }
58
120
  for (const moduleId of report.skippedLocked) {
59
121
  lines.push(` skipped ${moduleId}: another operation is in flight — retrying next tick`);
60
122
  }
@@ -20,7 +20,9 @@
20
20
  * resume alias for repair (acknowledges halt-on-recovery)
21
21
  */
22
22
 
23
+ import { spawnSync } from 'node:child_process';
23
24
  import {
25
+ BUS_VERSION,
24
26
  defineEvents,
25
27
  drainOnce,
26
28
  openBus,
@@ -37,10 +39,15 @@ import type { HookName } from '../../hooks/types';
37
39
  import type { ModuleManifest } from '../../manifest/schema';
38
40
  import type { EnsureRequiredPayload } from '../../services/bus-interview';
39
41
  import {
42
+ detectPlatform,
40
43
  installDaemon,
41
44
  planDaemonInstall,
42
45
  readInstalledUnit,
46
+ resolveRestartScope,
47
+ restartDaemon,
48
+ supervisorCommands,
43
49
  uninstallDaemon,
50
+ unitInstalledInAnyScope,
44
51
  } from '../../services/events-daemon';
45
52
  import { getArg, hasFlag } from '../parser';
46
53
  import type { CommandResult } from '../types';
@@ -897,6 +904,89 @@ export async function handleEventsUninstallDaemon(
897
904
  }
898
905
  }
899
906
 
907
+ /**
908
+ * `celilo events restart-daemon` — cycle the dispatcher through its supervisor
909
+ * and PROVE the new process is live on the installed code (celilo#604).
910
+ *
911
+ * The interesting case is the orphan: a dispatcher the supervisor doesn't own
912
+ * can't be killed by `systemctl restart`, and the one-dispatcher-per-bus guard
913
+ * (#584) then crash-loops the unit while the old code keeps serving. So this
914
+ * stops unmanaged dispatchers first, and verifies on the bus afterwards rather
915
+ * than trusting systemctl's exit code.
916
+ */
917
+ export async function handleEventsRestartDaemon(
918
+ _args: string[],
919
+ flags: Record<string, string | boolean>,
920
+ ): Promise<CommandResult> {
921
+ const bus = openCliBus();
922
+ try {
923
+ const platform = detectPlatform();
924
+ // No unit installed at all. apt-upgrade runs this on every box, including
925
+ // ones that never installed the daemon — failing those would break an
926
+ // upgrade that has nothing stale to fix. Distinguish the two cases:
927
+ // nothing running is a genuine no-op; something running is unsupervised
928
+ // and may be stale, and there is no supervisor to cycle it through.
929
+ if (!flags.system && !unitInstalledInAnyScope(platform)) {
930
+ const live = bus.liveDispatchers();
931
+ if (live.length === 0) {
932
+ return {
933
+ success: true,
934
+ message: 'No supervisor unit installed and no dispatcher running — nothing to restart.',
935
+ };
936
+ }
937
+ return {
938
+ success: false,
939
+ error: `A dispatcher is running unsupervised (pid ${live.map((d) => d.pid).join(', ')}, code v${live[0]?.version ?? '?'}) and no supervisor unit is installed, so it cannot be cycled — it may be serving stale code. Run \`celilo events install-daemon\`, enable the unit, then retry.`,
940
+ };
941
+ }
942
+ const scope = resolveRestartScope({ platform, scope: flags.system ? 'system' : undefined });
943
+ const cmds = supervisorCommands(platform, scope);
944
+ const result = await restartDaemon(
945
+ { platform, scope, expectedVersion: BUS_VERSION },
946
+ {
947
+ liveDispatchers: () =>
948
+ bus.liveDispatchers().map((d) => ({ pid: d.pid, version: d.version })),
949
+ supervisorPid: () => {
950
+ if (!cmds.mainPid) return null;
951
+ const out = spawnSync(cmds.mainPid[0], cmds.mainPid.slice(1), { encoding: 'utf-8' });
952
+ const pid = Number((out.stdout ?? '').trim());
953
+ return Number.isInteger(pid) && pid > 0 ? pid : null;
954
+ },
955
+ kill: (pid, signal) => {
956
+ try {
957
+ process.kill(pid, signal);
958
+ } catch {
959
+ // Already gone, or not ours to signal — the bus poll is the arbiter.
960
+ }
961
+ },
962
+ restartUnit: () => {
963
+ const out = spawnSync(cmds.restart[0], cmds.restart.slice(1), { encoding: 'utf-8' });
964
+ if (out.status !== 0) {
965
+ throw new Error(
966
+ `${cmds.restart.join(' ')} failed (exit ${out.status ?? 'signal'}): ${(out.stderr ?? '').trim()}`,
967
+ );
968
+ }
969
+ },
970
+ sleep: (ms) => new Promise((r) => setTimeout(r, ms)),
971
+ },
972
+ );
973
+ const lines = [
974
+ `Dispatcher restarted under the ${result.scope}-scope unit: ${shortenPath(result.unitPath)}`,
975
+ ` now running: pid ${result.dispatcher.pid}, code v${result.dispatcher.version}`,
976
+ ...(result.orphansKilled.length > 0
977
+ ? [
978
+ ` stopped ${result.orphansKilled.length} unsupervised dispatcher(s): pid ${result.orphansKilled.join(', ')}`,
979
+ ]
980
+ : []),
981
+ ];
982
+ return { success: true, message: lines.join('\n'), data: result };
983
+ } catch (err) {
984
+ return { success: false, error: err instanceof Error ? err.message : String(err) };
985
+ } finally {
986
+ bus.close();
987
+ }
988
+ }
989
+
900
990
  /**
901
991
  * `celilo events show-daemon` — print whatever unit file is currently
902
992
  * installed (`--system` for the system-scope unit) so the operator can