@celilo/cli 0.17.0 → 0.19.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CELILO_CORE_MODULES.md +1 -1
- package/CELILO_SUBSYSTEMS.md +38 -9
- package/drizzle/0019_backup_pid.sql +18 -0
- package/drizzle/meta/_journal.json +7 -0
- package/package.json +5 -5
- package/schemas/system_config.json +1 -1
- package/src/api/remote-client.test.ts +62 -0
- package/src/api/serve.ts +14 -6
- package/src/cli/command-tree-parser.ts +0 -1
- package/src/cli/commands/apt-upgrade.test.ts +20 -1
- package/src/cli/commands/apt-upgrade.ts +12 -2
- package/src/cli/commands/backup-sweep.ts +62 -0
- package/src/cli/commands/events.ts +90 -0
- package/src/cli/commands/module-operations.test.ts +45 -1
- package/src/cli/commands/module-operations.ts +35 -12
- package/src/cli/commands/module-show.ts +1 -0
- package/src/cli/commands/module-update.test.ts +72 -1
- package/src/cli/commands/module-update.ts +45 -22
- package/src/cli/commands/system-audit.ts +2 -0
- package/src/cli/commands/system-migrate.test.ts +56 -0
- package/src/cli/commands/system-migrate.ts +92 -4
- package/src/cli/commands/system-update.ts +5 -0
- package/src/cli/completion.ts +19 -0
- package/src/cli/fuel-gauge.ts +0 -1
- package/src/cli/generate-zsh-completion.ts +1 -1
- package/src/cli/index.ts +5 -1
- package/src/cli/tui/audit-state.ts +4 -0
- package/src/cli/tui/audit-tui.test.tsx +0 -1
- package/src/db/migration-status.test.ts +114 -0
- package/src/db/migration-status.ts +78 -0
- package/src/db/schema-introspection.ts +8 -1
- package/src/db/schema.ts +53 -9
- package/src/hooks/capability-loader.ts +30 -1
- package/src/ipam/allocator.ts +13 -3
- package/src/services/alerting/builtin-monitors.test.ts +42 -0
- package/src/services/alerting/builtin-monitors.ts +2 -0
- package/src/services/alerting/builtin-source.ts +15 -0
- package/src/services/audit/abandoned-operations.test.ts +73 -0
- package/src/services/audit/abandoned-operations.ts +0 -0
- package/src/services/audit/disk-space.test.ts +111 -0
- package/src/services/audit/disk-space.ts +114 -0
- package/src/services/audit/index.test.ts +1 -0
- package/src/services/audit/index.ts +9 -0
- package/src/services/audit/types.ts +2 -0
- package/src/services/backup-create.ts +4 -4
- package/src/services/backup-in-flight-refusal.test.ts +2 -0
- package/src/services/backup-metadata.ts +4 -0
- package/src/services/backup-staging.test.ts +134 -0
- package/src/services/backup-staging.ts +192 -0
- package/src/services/backup-sweep.test.ts +68 -0
- package/src/services/backup-sweep.ts +62 -0
- package/src/services/bus-interview.ts +11 -5
- package/src/services/config-interview.ts +1 -1
- package/src/services/deploy-ansible.ts +0 -1
- package/src/services/disk-probe.test.ts +74 -0
- package/src/services/disk-probe.ts +145 -0
- package/src/services/events-daemon.test.ts +244 -0
- package/src/services/events-daemon.ts +295 -8
- package/src/services/fleet-checks.test.ts +75 -4
- package/src/services/fleet-checks.ts +97 -12
- package/src/services/interview-errors.ts +20 -0
- package/src/services/module-operations.test.ts +22 -0
- package/src/services/module-operations.ts +48 -1
- package/src/services/module-subscriptions.test.ts +39 -6
- package/src/services/module-subscriptions.ts +6 -4
- package/src/services/module-types-generator.test.ts +6 -3
- package/src/services/module-types-generator.ts +12 -7
- package/src/services/remote-responder.test.ts +70 -0
- package/src/services/remote-responder.ts +27 -10
- package/src/services/responder-probe.ts +3 -1
- package/src/services/update/orchestrator.test.ts +1 -0
- package/src/variables/context.ts +6 -1
package/CELILO_CORE_MODULES.md
CHANGED
|
@@ -26,7 +26,7 @@ Each entry: `module id` — what it is — **provides** / **requires** capabilit
|
|
|
26
26
|
- **knot-unbound-internal** — split-horizon internal DNS via Knot (authoritative) + Unbound (recursive); lightweight, plain apt, no .NET. **provides:** `dns_internal`. Ships a base-module-aspect (`modules/knot-unbound-internal/base-module-aspect/`).
|
|
27
27
|
- **technitium** — internal split-horizon DNS resolver + authoritative server (web UI + HTTP API); heavier alternative to knot-unbound. **provides:** `dns_internal`. Ships a base-module-aspect (`modules/technitium/base-module-aspect/`).
|
|
28
28
|
- **namecheap** — public DNS A-record management via Namecheap Dynamic DNS API (HTTP, no browser automation). **provides:** `dns_registrar`.
|
|
29
|
-
- **wireguard** — owns the admin WireGuard tunnel on the firewall host: interface, listen port, peers (as records), and client subnet are module config rather than hand-maintained state. Exposes the listen port and **registers the client subnet as a trusted source**, so VPN reach into the managed zones is in the firewall registry and every converge re-emits it. Records `network.vpn.subnet`, which the internal resolver's split-horizon view also consumes. Adopts a running tunnel in place (existing key and peers retained; `wg syncconf`, never `wg-quick down`) because that tunnel is the operator's recovery path. **requires:** `firewall` (and the provider must support trusted-source registration — `iptables` does, `greenwave` does not).
|
|
29
|
+
- **wireguard** — owns the admin WireGuard tunnel on the firewall host: interface, listen port, peers (as records), and client subnet are module config rather than hand-maintained state. Exposes the listen port and **registers the client subnet as a trusted source**, so VPN reach into the managed zones is in the firewall registry and every converge re-emits it. Records `network.control-plane-vpn.subnet`, which the internal resolver's split-horizon view also consumes. Adopts a running tunnel in place (existing key and peers retained; `wg syncconf`, never `wg-quick down`) because that tunnel is the operator's recovery path. **requires:** `firewall` (and the provider must support trusted-source registration — `iptables` does, `greenwave` does not).
|
|
30
30
|
|
|
31
31
|
## Public edge (ingress / identity)
|
|
32
32
|
|
package/CELILO_SUBSYSTEMS.md
CHANGED
|
@@ -92,7 +92,7 @@ Module hooks reach a remote box ONLY through these typed primitives
|
|
|
92
92
|
- **`runAppCommand` / `runAppCommandWithSecret`** — escape-hatch on-box command; the secret variant feeds the secret on **stdin** (`$SECRET`), never argv.
|
|
93
93
|
- **`streamBackup` / `streamRestore` / `fetchFile` / `pushFile`** — binary-safe streaming via local shell redirect/pipe.
|
|
94
94
|
- **`waitFor`** — predicate-poll combinator.
|
|
95
|
-
- **`applyRenderedConfig`** — converge: write rendered config → validate → apply → rollback (one round-trip). Used by caddy (Caddyfile), knot (views), iptables (ruleset).
|
|
95
|
+
- **`applyRenderedConfig`** — converge: write rendered config → validate → apply → rollback (one round-trip). Used by caddy (Caddyfile), knot (views), iptables (ruleset). On success it RETAINS the file it replaced as `<path>.celilo-prev` — the durable "what celilo last rendered" record a whole-file converge diffs against.
|
|
96
96
|
- **`installAuthorizedKey`** — ONE-TIME credential bootstrap for an off-fleet account: `ssh-copy-id` under a password taken from the child ENV (`SSH_ASKPASS_REQUIRE=force`, so no `sshpass` dependency and the password never lands in a command string). Idempotent; used by `external_web` onboarding so every later publish is key-based.
|
|
97
97
|
- **`tailLog` / `grepLog`** — journald reads (regex / ignoreCase).
|
|
98
98
|
|
|
@@ -112,10 +112,34 @@ runner seam (`execRunner` real / `createMockRunner` for tests) lives in
|
|
|
112
112
|
(`renderRuleset`): registry + firewall state → a complete `iptables-restore`
|
|
113
113
|
file. Default-DROP FORWARD + established/related + egress + the coarse
|
|
114
114
|
zone-tier matrix (dmz→app, app→secure from `network.<zone>.subnet`) + per-service DNAT allows.
|
|
115
|
+
Egress permission and egress TRANSLATION are emitted together or neither: a
|
|
116
|
+
blanket `-o <wan>` MASQUERADE on a leaf, one `-s <subnet> -o <uplink>` rule per
|
|
117
|
+
network on a DOWNSTREAM firewall (no external edge). An upstream does not
|
|
118
|
+
translate on a downstream firewall's behalf.
|
|
119
|
+
`subnetsNeedingTranslation(state)` is that set — EVERY network behind the
|
|
120
|
+
firewall (tiers ∪ every declared zone subnet ∪ control plane ∪ registered
|
|
121
|
+
trusted sources) minus whichever of them CONTAINS the egress interface's own
|
|
122
|
+
address (`egressIp`, read off the box). Not just the data-plane tiers: a
|
|
123
|
+
downstream firewall's egress leg IS its LAN leg, so an untranslated network is
|
|
124
|
+
unreachable from a LAN host as well as from the internet.
|
|
115
125
|
- **Converge** — `modules/iptables/scripts/firewall-functions.ts` (`converge`):
|
|
116
|
-
`exposeService`/`unexposeService` register into the store, then render
|
|
117
|
-
atomically via `applyRenderedConfig`
|
|
118
|
-
|
|
126
|
+
`exposeService`/`unexposeService` register into the store, then render, CHECK,
|
|
127
|
+
DIFF, and only then apply atomically via `applyRenderedConfig`
|
|
128
|
+
(`iptables-restore`). Replaces the old per-rule `iptables -A`; the registry
|
|
129
|
+
(not `iptables -L`) is the source of truth. It does NOT apply unconditionally:
|
|
130
|
+
a failing check or an unexplainable removal throws, leaving the working
|
|
131
|
+
ruleset in place (`config.force` overrides, on the record).
|
|
132
|
+
- **Render-time completeness checks (pure)** — `modules/iptables/scripts/ruleset-checks.ts`
|
|
133
|
+
(`checkRenderedSet`, `blockingFindings`, `formatConvergeRefusal`): the SEMANTIC
|
|
134
|
+
counterpart to `iptables-restore --test`, which only validates syntax. Reports
|
|
135
|
+
a zone permitted to egress with no translation covering it, a control plane
|
|
136
|
+
that cannot reach a zone it manages, and an unknown control plane (a warning,
|
|
137
|
+
distinct from "known and absent"). Runs under `config.dryRun` too.
|
|
138
|
+
- **Pre-apply ruleset diff (pure)** — `modules/iptables/scripts/ruleset-diff.ts`
|
|
139
|
+
(`parseRuleset`, `diffRuleset`, `driftAgainstLive`, `snapshotCommand`,
|
|
140
|
+
`parseSnapshot`): what the converge would remove, and whether a registry change
|
|
141
|
+
explains it. Exact set algebra against the retained previous render, not a
|
|
142
|
+
heuristic. Live-vs-persisted drift is reported, never decided on.
|
|
119
143
|
- **Trusted-source registry (desired state)** — `trusted_sources` DB table
|
|
120
144
|
(migration `0016`) + `apps/celilo/src/services/trusted-sources.ts`
|
|
121
145
|
(`buildTrustedSourceStore`, `composeTrustedSubnets`). The sibling primitive to
|
|
@@ -144,7 +168,7 @@ runner seam (`execRunner` real / `createMockRunner` for tests) lives in
|
|
|
144
168
|
- **Deploy pipeline** — `apps/celilo/src/services/module-deploy.ts`.
|
|
145
169
|
- **DNS provider backfill** (re-emit registrations when a provider deploys) — `apps/celilo/src/services/dns-provider-backfill.ts` — `isDnsInternalProvider`, `backfillProviderDns`.
|
|
146
170
|
- **Base-module aspects (fan-out across the fleet)** — `apps/celilo/src/services/aspect-runner.ts` — `planAspectFanOut`, `runAspectFanOut`, `maybeRunAspectForTrigger`. Aspect content lives in `modules/<m>/base-module-aspect/` (e.g. knot-unbound-internal, technitium).
|
|
147
|
-
- **In-flight operation lock** — `apps/celilo/src/services/module-operations.ts` — `startOperation`/`completeOperation`/`failOperation` record deploy/uninstall/backup/restore in `module_operations`; `refuseIfInFlight`/`checkInFlight` are what backup and restore consult. Deploy and uninstall REGISTER but never check: it is a one-way guard protecting backup/restore consistency, not a general mutex (`openspec/specs/management-server-backup/spec.md` "In-flight operation refusal"). A row stops holding the lock once it is GONE, STOPPED/zombie (`isPidRunnable`, `ps -o state=` — `kill(pid,0)` calls a Ctrl-Z'd process alive), or older than `OPERATION_TTL_MS` (2h). The TTL is not redundancy: a pid is a recycled number, and once the pid space wraps an old row names an unrelated healthy process. Operator surface: `celilo module operations [list|clear] [--all]` (`apps/celilo/src/cli/commands/module-operations.ts`).
|
|
171
|
+
- **In-flight operation lock** — `apps/celilo/src/services/module-operations.ts` — `startOperation`/`completeOperation`/`failOperation` record deploy/uninstall/backup/restore in `module_operations`; `refuseIfInFlight`/`checkInFlight` are what backup and restore consult. Deploy and uninstall REGISTER but never check: it is a one-way guard protecting backup/restore consistency, not a general mutex (`openspec/specs/management-server-backup/spec.md` "In-flight operation refusal"). A row stops holding the lock once it is GONE, STOPPED/zombie (`isPidRunnable`, `ps -o state=` — `kill(pid,0)` calls a Ctrl-Z'd process alive), or older than `OPERATION_TTL_MS` (2h). The TTL is not redundancy: a pid is a recycled number, and once the pid space wraps an old row names an unrelated healthy process. Operator surface: `celilo module operations [list|clear] [--abandoned] [--all]` (`apps/celilo/src/cli/commands/module-operations.ts`); `list` shows only what holds the lock, abandoned rows are summarised unless `--abandoned`. Abandoned rows are reclaimed hourly by the `celilo-operations-sweep` bus subscriber (`timer.tick.1h` → `celilo module operations clear`, armed by `ensureOperationsSweepSubscriber` from module registration and `celilo system migrate`). `clear` MARKS rows failed rather than deleting them, and that is load-bearing: the `abandoned_operations` audit reads exactly those released rows to notice one module's operation dying over and over.
|
|
148
172
|
- **Backup artifact encryption** — `apps/celilo/src/services/backup-cipher.ts` — `encryptFileToFile`/`decryptFileToFile`, file-in/file-out and streamed, used by every backup writer (`backup-create.ts`) and reader (`backup-restore.ts`, `restore-from-file.ts`). Do NOT route artifacts through `secrets/encryption.ts`: that API is string-in/string-out for short DB values, and feeding it a tar cost base64 (1.33x) then hex (2x) then `JSON.stringify` — ~9x the artifact in memory, which OOM-killed forgejo's 774 MB backup, and a hard ~805 MB ceiling from the max string length that no amount of RAM raises. On-disk format is `magic "CELILOBK" (8) | version (1) | iv (16) | ciphertext | GCM tag (16)`; the tag is a trailer because it does not exist until the last byte is encrypted. `decryptFileToFile` still reads the pre-2026-07 JSON-envelope artifacts, discriminating on the magic bytes — the envelope's own `schemaVersion` cannot serve, as it lives inside the encrypted tar.
|
|
149
173
|
|
|
150
174
|
## Generation & templating
|
|
@@ -174,7 +198,8 @@ is currently wrong, and routes carry the message to a person's phone. Design:
|
|
|
174
198
|
- **Alert identity** — `apps/celilo/src/services/alerting/keys.ts` — the key grammar (`module:<id>[/check:<name>]`, `builtin:<check>[/<kind>:<target>]`) that makes "the same problem" the same alert across runs. `moduleAlertKey`, `moduleCheckAlertKey`, `builtinAlertKey`, `parseAlertKey`.
|
|
175
199
|
- **Reconciliation** — `apps/celilo/src/services/alerting/reconcile.ts` (`reconcile`) — a successful run's failing-key set is authoritative and resolution is by SET DIFFERENCE (absent ⇒ resolved). A run whose outcome is `error` resolves NOTHING and fires a module-level alert instead: the false-all-clear guard.
|
|
176
200
|
- **Monitor execution** — `apps/celilo/src/services/alerting/run-monitor.ts` (`runOneMonitor`) + `sweep.ts` (`selectDueMonitors`) + `builtin-monitors.ts` / `health-coverage.ts` (the built-in checks and the "module with no health check" coverage check).
|
|
177
|
-
- **Scheduled audit categories (`builtin_check` monitors)** — `apps/celilo/src/services/alerting/builtin-source.ts` — `SCHEDULABLE_BUILTIN_CHECKS` is the list of `celilo system audit` categories cheap enough to run every sweep: `machines_reachable` and `
|
|
201
|
+
- **Scheduled audit categories (`builtin_check` monitors)** — `apps/celilo/src/services/alerting/builtin-source.ts` — `SCHEDULABLE_BUILTIN_CHECKS` is the list of `celilo system audit` categories cheap enough to run every sweep: `machines_reachable`, `backups`, `disk_space`, and `abandoned_operations` (`apps/celilo/src/services/audit/abandoned-operations.ts` — ≥3 abandonments of the same (module, operation) in 7d, the fingerprint of an operation being killed mid-flight). Everything else in the audit needs the whole world injected (proxmox, terraform, registry) and is not schedulable. Enable one with `celilo monitor add backups --interval 1h`. Targets are tab-completable — `completion.ts` reads `SCHEDULABLE_BUILTIN_CHECKS` directly rather than a hand-copied list, so a newly-schedulable check is completable immediately.
|
|
202
|
+
- **Disk-space check** — `apps/celilo/src/services/audit/disk-space.ts` (`auditDiskSpace`, pure over measurements) + `apps/celilo/src/services/disk-probe.ts` (`probeDiskUsage`). Thresholds: `drift` at 85%, `blocked` at 95% — early enough to act on, since a check that fires at exhaustion reports an outage rather than preventing one. ⚠️ **The local management box is MEASURED, not exempted.** `probeMachines()` deliberately reports the local box reachable without probing it (celilo has no SSH key for itself, and the question is meaningless there); copying that shortcut into a disk check would skip the host most likely to fill — the one that stages backups, caches modules and writes the logs, and the one that DID fill. Local reads `statfs`; remote runs `df -P /` over the same bounded SSH. `percentUsed` matches `df`'s capacity semantics (excludes root-reserved blocks) so an alert and an operator's own `df` agree. An unmeasurable host yields a `todo` finding — recorded, never paged, because `machines_reachable` is already paging for that host. Findings are subjected on the **hostname**, not the machine UUID, because suppression resolves a machine's ancestor key from the hostname (see #596, where `machines_reachable` gets this wrong and its alerts therefore never suppress anything). The `backups` roster comes from `apps/celilo/src/services/audit/backup-source.ts` (`loadBackupAuditInfo`), shared with `celilo system audit` so both judge the same fleet.
|
|
178
203
|
- **The sweep** — `apps/celilo/src/services/alerting/sweep-runner.ts` (`runSweep`) — the ordered pass that makes alerting run by itself: run due monitors → promote past-grace alerts → re-evaluate suppression → flush quiet-hours deferrals → notify. Driven by `celilo alerts sweep` on `timer.tick.5m`. Never throws for one bad monitor.
|
|
179
204
|
- **Suppression (topology-derived, never configured)** — `apps/celilo/src/services/alerting/suppression.ts` — `ancestorKeysFor`/`findSuppressor`/`machineAlertKey`. A firing machine explains its modules' failures; a firing capability provider explains its zone's consumers. Derived from `module_systems`, so it cannot drift from reality. Deploy windows: `deploy-hooks.ts` (`openDeployWindow`/`closeDeployWindows` — closed by module, so a crashed deploy self-heals).
|
|
180
205
|
- **Escalation & quiet hours** — `escalation.ts` (`decideEscalation`, every reason to stay silent enumerated) + `quiet-hours.ts` (`isWithinQuietHours`, Intl-based and DST-safe). Quiet hours defer the MESSAGE while the escalation clock keeps running.
|
|
@@ -194,7 +219,9 @@ Creation, scheduling and freshness. A module declares an `on_backup` hook and a
|
|
|
194
219
|
|
|
195
220
|
- **Creation** — `apps/celilo/src/services/backup-create.ts` — `createModuleBackup` (invokes the module's `on_backup` hook into an encrypted envelope), `createSystemStateBackup` (celilo.db), `findBackupEligibleModules`, `isBackupDue`. Storage destinations: `backup-storage.ts`. Retention: `backup-retention.ts` (`pruneBackupsForModule`). Restore: `backup-restore.ts`.
|
|
196
221
|
- **Cadence (one accessor)** — `apps/celilo/src/services/backup-schedule.ts` — `effectiveBackupSchedule(manifest)`. An absent `backup.schedule` means `daily`, NOT `manual`; opting out takes an explicit `manual`. Both the freshness audit and the backup sweep must read cadence through this one function, or a module can be alerted-on but never backed up.
|
|
197
|
-
- **The sweep** — `apps/celilo/src/services/backup-sweep.ts` — `runBackupSweep` (the pass that makes backups run by themselves: for each eligible module, is its declared cadence due → back it up → apply declared retention) + `ensureBackupSweepSubscriber`. Driven by `celilo backup sweep` on `timer.tick.1h` — the coarsest tick that can still serve an `hourly` cadence. Armed
|
|
222
|
+
- **The sweep** — `apps/celilo/src/services/backup-sweep.ts` — `runBackupSweep` (the pass that makes backups run by themselves: reclaim orphaned staging → for each eligible module, is its declared cadence due → back it up → apply declared retention) + `ensureBackupSweepSubscriber`. Driven by `celilo backup sweep` on `timer.tick.1h` — the coarsest tick that can still serve an `hourly` cadence. Armed from BOTH `registerModuleSubscriptions` (any module declaring an `on_backup` hook, so it appears on install or `module update`) AND `celilo system migrate`, which the `.deb` postinst runs on every apt upgrade — registering only from module install/update meant a corrected budget never reached an existing fleet, since the row already exists and module updates can be weeks apart. A run refused by the in-flight operation lock is a skip retried next tick, never a failure.
|
|
223
|
+
- **⚠️ The sweep's budget is stated, never inherited** — `BACKUP_SWEEP_TIMEOUT_MS` (4h) and `BACKUP_SWEEP_MAX_ATTEMPTS` (1) in `backup-sweep.ts`. The event bus defaults to `timeout_ms: 60000` / `max_attempts: 3`, and inheriting them made scheduled backups structurally impossible: one forgejo backup measured ~5.5 minutes (1.3 GB result, 3.9 GB peak staging), so the dispatcher SIGTERMed it at 60s — three times an hour, for days. The retry count is half the bug, not a detail: an impossible pass retried 3x strands 3x the staging (27 GB in 5.7 hours on celilo-mgr). The hourly tick IS the retry.
|
|
224
|
+
- **Staging reclamation** — `apps/celilo/src/services/backup-staging.ts` — `reapOrphanedStaging`, `stagingDirFor` (the single source of truth for `/tmp/celilo-backup-<record.id>`, shared with `backup-create.ts` so writer and reaper cannot drift). `backup-create.ts` removes its staging in a `finally`, which is correct and NOT enough: a `finally` never runs when the process is killed by a signal — dispatcher timeout, OOM, Ctrl-C, reboot — and those strand the LARGEST directories. So reclamation is kill-mode agnostic by construction: it asks "is anyone still using this?", answered from the `backups.pid` column plus `isPidRunnable`, both of which outlive the process. A directory is removed only when its owner is provably gone (record absent, record terminal, pid dead, or past `STAGING_TTL_MS` = 6h — the TTL is the only check surviving pid reuse). A live backup is always kept; an unrecognised name is ignored, never deleted. Reclaiming a record that still claimed `in_progress` also marks it failed with `ABANDONED_BACKUP_MESSAGE`, so `celilo backup list` stops showing phantom in-flight backups and the `backups` drift check cannot read a dead attempt as a fresh backup.
|
|
198
225
|
- **Freshness audit** — `apps/celilo/src/services/audit/backups.ts` (`auditBackups`) — `backup_missing` / `backup_stale` drift findings against the same declared cadence.
|
|
199
226
|
- **CLI** — `apps/celilo/src/cli/commands/` — `backup-sweep.ts`, `backup-create.ts` (also `celilo module backup`), `backup-list.ts`, `backup-restore.ts`, `backup-prune.ts`, `backup-delete.ts`, `backup-import.ts`, `backup-pull.ts`, `backup-name.ts`.
|
|
200
227
|
- **Storage destinations CLI** — `storage-add-local.ts`, `storage-add-s3.ts`, `storage-list.ts`, `storage-verify.ts`, `storage-set-default.ts`, `storage-set-path.ts` (relocate a local destination, migrating existing archives unless `--no-migrate`), `storage-remove.ts`. Any credential change goes through `updateStorageCredentials` in `backup-storage.ts`, which clears the verification stamp — a `✓ Verified` must never describe a destination it was not measured against (#566).
|
|
@@ -205,7 +232,9 @@ Creation, scheduling and freshness. A module declares an `on_backup` hook and a
|
|
|
205
232
|
|
|
206
233
|
## Events
|
|
207
234
|
|
|
208
|
-
- **
|
|
235
|
+
- **Migration interrogation** — `apps/celilo/src/db/migration-status.ts` — `getMigrationStatus(sqlite, migrationsFolder)` reports applied count, latest applied migration BY TAG, and pending ones by name (joining `__drizzle_migrations.created_at` to drizzle journal `when`, which is exact). Surface: `celilo system migrate --status`, which opens the DB **read-only** on purpose — `getDb()` auto-migrates on open, so a status routed through it would repair what it claims to report and could never say "pending". Paired with `findSchemaDrift` (`db/schema-introspection.ts`), which is column-aware: a table COUNT cannot distinguish "the column migration applied" from "nothing happened", which is why a rollout asserting `backups.pid` had to reach for `sqlite3` over SSH. `checkSchemaDrift` (`services/fleet-checks.ts`) fails on a missing table, a missing column, OR an unapplied journal migration, and its summary names tables AND columns so the operator can see what was checked.
|
|
236
|
+
- **Event bus** — `packages/event-bus/src/index.ts` — `Bus`, `openBus`, `defineEvents`, `defineHandler`, `runDispatcher`, pattern matching + timer ticks (`emitDueTimerTicks`, `retentionSweep`). **Exactly one dispatcher per bus**: `runDispatcher` refuses to start while another dispatcher's process is alive (`assertSoleDispatcher` in `dispatcher.ts`, liveness via `bus.liveDispatchers()` — `kill(pid,0)`, not heartbeat age, since a tick blocks for as long as its slowest handler). The exclusion is here rather than in the systemd unit because a stranded dispatcher can sit outside the unit's cgroup where `KillMode` cannot reach it (#580). `bus.health()` reports `dispatcherCount`/`dispatchers` and a `duplicate_dispatcher` status; `checkDispatcher` (`services/fleet-checks.ts`) fails on more than one. **Supervision** (`services/events-daemon.ts`): the unit is named `celilo-events.service` in BOTH the user and system scope, so the two are indistinguishable in every operator-facing string — `install-daemon` (which defaults to **user** scope) therefore refuses when the other scope's unit exists, and `checkDispatcher` compares the live pid against each installed unit's `MainPID` (`unitMainPid`) rather than merely testing that a unit file exists, since a file nobody is running is not supervision (#610).
|
|
237
|
+
- **Dispatcher supervision (install / restart)** — `apps/celilo/src/services/events-daemon.ts` — `installDaemon`/`planDaemonInstall`/`uninstallDaemon`/`readInstalledUnit` write the systemd unit or launchd plist without touching supervisor state, and `restartDaemon` (+ pure `orphanDispatcherPids`, `resolveRestartScope`, `supervisorCommands`) is the one verb that DOES cycle it. CLI: `celilo events install-daemon|uninstall-daemon|show-daemon|restart-daemon [--system]`. `restart-daemon` exists because the dispatcher runs the code it LOADED: celilo-mgr sat 9 days on event-bus v0.1.8 after apt installed v0.2.0, running the very bug the release fixed (celilo#604). Two things make it non-trivial and both are load-bearing: (1) it stops any live dispatcher the supervisor does not own first — an ORPHAN (PPID 1) is invisible to `systemctl restart`, and `assertSoleDispatcher` then crash-loops the unit while the old code keeps serving; (2) it verifies on the BUS that a NEW pid is live reporting `BUS_VERSION`, never off systemctl's exit code, which returns 0 into exactly that crash loop. System scope shells `sudo systemctl` (the unit is root-owned; celilo runs unprivileged), covered by the scoped `/etc/sudoers.d/celilo-events-restart` conffile that `celilo-bootstrap` ships — a unit test asserts the argv and the grant cannot drift apart.
|
|
209
238
|
|
|
210
239
|
## Remote API (drive the CLI over the wire)
|
|
211
240
|
|
|
@@ -219,7 +248,7 @@ Run any celilo command on celilo-mgr over SSH instead of screen-scraping `ssh <h
|
|
|
219
248
|
- **Access control** — `apps/celilo/src/services/api-access.ts` — `grantPrincipal`, `isAuthorized` (deny-by-default, `command:subcommand` grants), `renderAuthorizedKeys`. Table: `api_principals` (`apps/celilo/src/db/schema.ts`). CLI: `apps/celilo/src/cli/commands/api.ts` (`api grant|list|revoke|authorized-keys|key new`).
|
|
220
249
|
- **Mid-run interview bridge (`kind:daemon` responder)** — `apps/celilo/src/services/remote-responder.ts` — `startRemoteResponder` bridges bus `interview.required.*` ↔ wire.
|
|
221
250
|
- **Server provisioning** — the `celilo-bootstrap` deb (`packaging/celilo-bootstrap/scripts/postinst`) creates the non-root `celilo-api` landing account + sshd; membership in the `celilo` group + `/etc/sudoers.d/celilo` (`!use_pty`) gives api-serve DB access via the wrapper's sudo-drop.
|
|
222
|
-
- **Self-upgrade (apt)** — `celilo apt-upgrade` (`apps/celilo/src/cli/commands/apt-upgrade.ts`) upgrades the deb-installed `celilo`/`celilo-bootstrap` packages (`apt-get update` → `--only-upgrade install`) then spawns a fresh `celilo system migrate` (ISS-0100). It's the RW target behind the MCP's registry-derived `celilo_apt_upgrade` tool; the celilo user's two apt invocations are scoped-sudo'd by `/etc/sudoers.d/celilo-apt-upgrade`, shipped by `celilo-bootstrap`. **This upgrades celilo ITSELF — not the modules it manages. For those, see Module auto-upgrade below; the two are routinely confused.**
|
|
251
|
+
- **Self-upgrade (apt)** — `celilo apt-upgrade` (`apps/celilo/src/cli/commands/apt-upgrade.ts`) upgrades the deb-installed `celilo`/`celilo-bootstrap` packages (`apt-get update` → `--only-upgrade install`) then spawns a fresh `celilo system migrate` (ISS-0100), then `celilo events restart-daemon` so the dispatcher actually runs the code just installed — a failure there fails the whole command and names which steps DID complete, because "upgraded" while the dispatcher serves stale code is the silent state celilo#604 documents. It's the RW target behind the MCP's registry-derived `celilo_apt_upgrade` tool; the celilo user's two apt invocations are scoped-sudo'd by `/etc/sudoers.d/celilo-apt-upgrade`, shipped by `celilo-bootstrap`. **This upgrades celilo ITSELF — not the modules it manages. For those, see Module auto-upgrade below; the two are routinely confused.**
|
|
223
252
|
- **Module auto-upgrade (registry-poll CD)** — the *pull* half of continuous deployment: celilo-mgr polls the registry and upgrades opted-in modules unattended. Spec: `openspec/specs/module-auto-upgrade/spec.md`. Entry points: `apps/celilo/src/cli/commands/module-upgrade.ts` — `runRegistryPoll` (the `--poll` path), `selectPollTargets` (pure: `autoUpgrade && latest && change ∉ {up-to-date, ahead}`), `upgradeOneModule` (update → backup → deploy → verify), `needsPreUpgradeBackup`, `pickAutoUpgrade`/`pickUpgradePolicy` (both fail closed/safe); `classifyVersionChange` in `module-update.ts` (treats a registry `+N` revision as a patch); `resolveDeployPosture` in `apps/celilo/src/services/deploy-posture.ts`. Trigger: celilo-mgmt's `registry-poll` subscription (`modules/celilo-mgmt/manifest.yml`) on `timer.tick.15m` with handler **`celilo module upgrade --poll`** — the flag is REQUIRED, since the dispatcher appends the event id positionally and a bare handler would consume it as the optional module name (silent: 3108 deliveries, 0 successes). Operator controls are framework config keys settable on ANY module (`FRAMEWORK_CONFIG_KEYS` in `module-config.ts`): `auto_upgrade` (opt-in, default false) and `upgrade_policy` (`by-semver`|`always-safe`|`always-fast`), validated at set time because both readers fail open. ⚠️ `always-safe` guarantees safe *posture*, NOT a backup — `needsPreUpgradeBackup` also requires the TARGET manifest to declare an `on_backup` hook, else it warns and proceeds. Confirm a data-bearing module declares `on_backup` before enabling `auto_upgrade` on it. The *build* half (app CI publishing a `.netapp` on merge) is not yet shipped — `openspec/changes/build-bus-poll-cd`.
|
|
224
253
|
- **MCP service (`@celilo/mcp`)** — `packages/mcp/src/` — an operator-facing stdio MCP server (official `@modelcontextprotocol/sdk`, bin `celilo-mcp`) that drives a remote celilo server over the Remote API for an AI client. Two-item config (`config.ts`: `server` + `defaultUser`, env or `~/.config/celilo-mcp/config.json`). Dual-principal auth (`auth.ts`: `celilo-mcp auth setup` enrolls read-only `celilo-mcp-ro` + full `celilo-mcp-rw` ed25519 keypairs, prints the exact `celilo api grant` lines the operator runs server-side). Transport (`transport.ts`): reuses `@celilo/core` `runRemoteClient`, selecting the principal by `ssh -i <key>` and capturing structured output. Tool surface is generated LIVE from the server's command registry — `registry-fetch.ts` fetches `celilo commands --json` (+ `service list --json` for configured providers) over the RO principal on connect; `tools-from-registry.ts` (pure) projects that into one tool per runnable leaf, grouped by top-level command (`celilo_module_*`, `celilo_proxmox_*`, …), each with a Zod input schema from the leaf's args/flags and a read/write tag → RO/RW routing, plus a generic `celilo_run` escape hatch. Auto-detect hides provider-gated groups (e.g. `celilo_proxmox_*` until a Proxmox service is configured) and re-detects on a timer, emitting `notifications/tools/list_changed` when the surface changes. Coverage gate (`tests/coverage.test.ts`) asserts every registry leaf maps to a tool. Composite RO troubleshooting tools (`troubleshoot.ts` pure correlation + `troubleshoot-tools.ts` thin adapters): `celilo_assess_module <id>` and `celilo_fleet_status` correlate `celilo audit --json` (the drift backbone) with the `module list --json` roster into a per-module / fleet-wide verdict. Design: `openspec/changes/celilo-mcp-service/proposal.md`. (Distinct from the dev/ops `@celilo/mcp-server` below.)
|
|
225
254
|
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
-- Record which process owns a backup's staging directory.
|
|
2
|
+
--
|
|
3
|
+
-- `backup-create.ts` assembles every envelope under `/tmp/celilo-backup-<id>`
|
|
4
|
+
-- and removes it in a `finally`. A `finally` does not run when the process is
|
|
5
|
+
-- killed by a signal — a dispatcher timeout, an OOM, a reboot — and those are
|
|
6
|
+
-- exactly the cases that strand the largest directories. On celilo-mgr the
|
|
7
|
+
-- scheduled sweep was killed at 60s against a backup needing ~5.5 minutes,
|
|
8
|
+
-- three times an hour, stranding 27 GB in under six hours.
|
|
9
|
+
--
|
|
10
|
+
-- Reclamation therefore cannot depend on how a backup ends. The staging reaper
|
|
11
|
+
-- (services/backup-staging.ts) asks "is anyone still using this directory?"
|
|
12
|
+
-- instead, and this column is what lets it answer: the directory names its
|
|
13
|
+
-- record, and the record names its process.
|
|
14
|
+
--
|
|
15
|
+
-- Nullable on purpose. Rows written before this column exists have no pid; the
|
|
16
|
+
-- reaper keeps their staging until the TTL expires rather than guessing.
|
|
17
|
+
|
|
18
|
+
ALTER TABLE `backups` ADD `pid` integer;
|
|
@@ -134,6 +134,13 @@
|
|
|
134
134
|
"when": 1783400000000,
|
|
135
135
|
"tag": "0018_drop_alert_policy_snapshot",
|
|
136
136
|
"breakpoints": true
|
|
137
|
+
},
|
|
138
|
+
{
|
|
139
|
+
"idx": 19,
|
|
140
|
+
"version": "6",
|
|
141
|
+
"when": 1783500000000,
|
|
142
|
+
"tag": "0019_backup_pid",
|
|
143
|
+
"breakpoints": true
|
|
137
144
|
}
|
|
138
145
|
]
|
|
139
146
|
}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@celilo/cli",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.19.0",
|
|
4
4
|
"description": "Celilo — home lab orchestration CLI",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"bin": {
|
|
@@ -57,10 +57,10 @@
|
|
|
57
57
|
},
|
|
58
58
|
"dependencies": {
|
|
59
59
|
"@aws-sdk/client-s3": "^3.1024.0",
|
|
60
|
-
"@celilo/capabilities": "^0.
|
|
61
|
-
"@celilo/cli-display": "^0.1.
|
|
62
|
-
"@celilo/core": "^0.
|
|
63
|
-
"@celilo/event-bus": "^0.
|
|
60
|
+
"@celilo/capabilities": "^0.10.0",
|
|
61
|
+
"@celilo/cli-display": "^0.1.10",
|
|
62
|
+
"@celilo/core": "^0.4.0",
|
|
63
|
+
"@celilo/event-bus": "^0.2.0",
|
|
64
64
|
"@clack/prompts": "^1.1.0",
|
|
65
65
|
"ajv": "^8.18.0",
|
|
66
66
|
"drizzle-orm": "^0.36.4",
|
|
@@ -74,7 +74,7 @@
|
|
|
74
74
|
"maximum": 4094,
|
|
75
75
|
"description": "VLAN tag for internal zone (not defaulted; internal is untagged)"
|
|
76
76
|
},
|
|
77
|
-
"network.vpn.subnet": {
|
|
77
|
+
"network.control-plane-vpn.subnet": {
|
|
78
78
|
"type": "string",
|
|
79
79
|
"pattern": "^\\d{1,3}\\.\\d{1,3}\\.\\d{1,3}\\.\\d{1,3}/\\d{1,2}$",
|
|
80
80
|
"description": "WireGuard VPN client subnet CIDR (remote-access tunnel; consumed by technitium's split-horizon → in-zone caddy view, #275, and by the firewall's trusted sources once the wireguard module registers it)"
|
|
@@ -89,3 +89,65 @@ test('renders a forwarded interview and sends the answer back', async () => {
|
|
|
89
89
|
expect(answer).toBeDefined();
|
|
90
90
|
expect(JSON.parse(answer as string)).toEqual({ type: 'answer', id: 'q1', value: 'myhost' });
|
|
91
91
|
});
|
|
92
|
+
|
|
93
|
+
/**
|
|
94
|
+
* Regression for the fabricated "operator declined".
|
|
95
|
+
*
|
|
96
|
+
* The default renderer prompts with clack, which reads keypresses off stdin
|
|
97
|
+
* whether or not stdin is a terminal. Driven over the MCP (stdin = the JSON-RPC
|
|
98
|
+
* stream) the next newline submitted the prompt at its `initialValue` — the
|
|
99
|
+
* question's `defaultValue` — so a breaking update nobody saw came back as a
|
|
100
|
+
* considered "no". With no terminal the client must say it cannot answer.
|
|
101
|
+
*/
|
|
102
|
+
test('no TTY and no renderer → answers with an error, never the default', async () => {
|
|
103
|
+
const writes: string[] = [];
|
|
104
|
+
const encoder = new TextEncoder();
|
|
105
|
+
let controller!: ReadableStreamDefaultController<Uint8Array>;
|
|
106
|
+
const stdout = new ReadableStream<Uint8Array>({
|
|
107
|
+
start(c) {
|
|
108
|
+
controller = c;
|
|
109
|
+
},
|
|
110
|
+
});
|
|
111
|
+
const push = (obj: unknown) => controller.enqueue(encoder.encode(`${JSON.stringify(obj)}\n`));
|
|
112
|
+
|
|
113
|
+
const transport: RemoteTransport = {
|
|
114
|
+
stdin: {
|
|
115
|
+
write(chunk: string) {
|
|
116
|
+
writes.push(chunk);
|
|
117
|
+
if (chunk.includes('"answer"')) {
|
|
118
|
+
push({ type: 'result', success: false, exitCode: 1 });
|
|
119
|
+
controller.close();
|
|
120
|
+
}
|
|
121
|
+
},
|
|
122
|
+
},
|
|
123
|
+
stdout,
|
|
124
|
+
kill() {},
|
|
125
|
+
exited: Promise.resolve(1),
|
|
126
|
+
};
|
|
127
|
+
|
|
128
|
+
push({ type: 'ready', protocolVersion: 1 });
|
|
129
|
+
push({
|
|
130
|
+
type: 'interview',
|
|
131
|
+
id: 'q1',
|
|
132
|
+
scope: 'module-upgrade:iptables',
|
|
133
|
+
key: 'apply_breaking',
|
|
134
|
+
kind: 'confirm',
|
|
135
|
+
message: 'Apply breaking update for iptables (1.0.2+9 → 2.0.0+1)?',
|
|
136
|
+
defaultValue: 'false',
|
|
137
|
+
});
|
|
138
|
+
|
|
139
|
+
// No `renderInterview` — exactly what runRemoteCapture used to do. bun test
|
|
140
|
+
// runs with a piped stdin, i.e. the MCP server's situation.
|
|
141
|
+
expect(process.stdin.isTTY).toBeFalsy();
|
|
142
|
+
const code = await runRemoteClient('ignored', ['module', 'update'], {
|
|
143
|
+
openTransport: () => transport,
|
|
144
|
+
out: { write() {} },
|
|
145
|
+
});
|
|
146
|
+
|
|
147
|
+
expect(code).toBe(1);
|
|
148
|
+
const answer = writes.find((w) => w.includes('"answer"'));
|
|
149
|
+
expect(answer).toBeDefined();
|
|
150
|
+
const parsed = JSON.parse(answer as string) as { value: unknown; error?: string };
|
|
151
|
+
expect(parsed.value).not.toBe(false);
|
|
152
|
+
expect(parsed.error).toContain('module-upgrade:iptables.apply_breaking');
|
|
153
|
+
});
|
package/src/api/serve.ts
CHANGED
|
@@ -108,14 +108,19 @@ export async function apiServeMode(principal: string): Promise<void> {
|
|
|
108
108
|
send({ type: 'ready', protocolVersion: API_PROTOCOL_VERSION });
|
|
109
109
|
|
|
110
110
|
const busDbPath = getEventBusPath();
|
|
111
|
-
const pendingAnswers = new Map<
|
|
111
|
+
const pendingAnswers = new Map<
|
|
112
|
+
string,
|
|
113
|
+
{ resolve: (value: unknown) => void; reject: (error: Error) => void }
|
|
114
|
+
>();
|
|
112
115
|
|
|
113
116
|
const ask = (interview: WireInterview): Promise<unknown> =>
|
|
114
|
-
new Promise((resolve) => {
|
|
115
|
-
pendingAnswers.set(interview.id, resolve);
|
|
117
|
+
new Promise((resolve, reject) => {
|
|
118
|
+
pendingAnswers.set(interview.id, { resolve, reject });
|
|
116
119
|
send({
|
|
117
120
|
type: 'interview',
|
|
118
121
|
id: interview.id,
|
|
122
|
+
scope: interview.scope,
|
|
123
|
+
key: interview.key,
|
|
119
124
|
kind: interview.kind,
|
|
120
125
|
message: interview.message,
|
|
121
126
|
description: interview.description,
|
|
@@ -140,10 +145,13 @@ export async function apiServeMode(principal: string): Promise<void> {
|
|
|
140
145
|
}
|
|
141
146
|
|
|
142
147
|
if (msg.type === 'answer') {
|
|
143
|
-
const
|
|
144
|
-
if (
|
|
148
|
+
const pending = pendingAnswers.get(msg.id);
|
|
149
|
+
if (pending) {
|
|
145
150
|
pendingAnswers.delete(msg.id);
|
|
146
|
-
|
|
151
|
+
// `error` = the client couldn't reach a decider. Fail the waiting
|
|
152
|
+
// command rather than letting the question fall back to its default.
|
|
153
|
+
if (msg.error) pending.reject(new Error(msg.error));
|
|
154
|
+
else pending.resolve(msg.value);
|
|
147
155
|
}
|
|
148
156
|
continue;
|
|
149
157
|
}
|
|
@@ -163,7 +163,6 @@ export class CommandTreeParser {
|
|
|
163
163
|
* Strip ANSI color codes from text
|
|
164
164
|
*/
|
|
165
165
|
private stripAnsiCodes(text: string): string {
|
|
166
|
-
// biome-ignore lint/suspicious/noControlCharactersInRegex: ANSI escape codes require control characters
|
|
167
166
|
return text.replace(/\x1B\[[0-9;]*[a-zA-Z]/g, '');
|
|
168
167
|
}
|
|
169
168
|
|
|
@@ -2,7 +2,7 @@ import { describe, expect, test } from 'bun:test';
|
|
|
2
2
|
import { handleAptUpgrade } from './apt-upgrade';
|
|
3
3
|
|
|
4
4
|
describe('handleAptUpgrade', () => {
|
|
5
|
-
test('runs
|
|
5
|
+
test('runs every step in order when each succeeds, ending with the dispatcher restart', async () => {
|
|
6
6
|
const seen: string[][] = [];
|
|
7
7
|
const result = await handleAptUpgrade([], {}, (argv) => {
|
|
8
8
|
seen.push(argv);
|
|
@@ -14,9 +14,28 @@ describe('handleAptUpgrade', () => {
|
|
|
14
14
|
['sudo', 'apt-get', 'update'],
|
|
15
15
|
['sudo', 'apt-get', '-y', '--only-upgrade', 'install', 'celilo', 'celilo-bootstrap'],
|
|
16
16
|
['/usr/local/bin/celilo', 'system', 'migrate'],
|
|
17
|
+
// celilo#604: without this, apt installs new code and the running
|
|
18
|
+
// dispatcher keeps serving the old — celilo-mgr did so for 9 days.
|
|
19
|
+
['/usr/local/bin/celilo', 'events', 'restart-daemon'],
|
|
17
20
|
]);
|
|
18
21
|
});
|
|
19
22
|
|
|
23
|
+
test('fails, and names what DID complete, when the dispatcher restart fails', async () => {
|
|
24
|
+
const result = await handleAptUpgrade([], {}, (argv) => ({
|
|
25
|
+
status: argv.includes('restart-daemon') ? 1 : 0,
|
|
26
|
+
}));
|
|
27
|
+
|
|
28
|
+
// The packages ARE upgraded and the dispatcher is NOT on the new code.
|
|
29
|
+
// Saying so in the failure is the acceptance condition — silence here is
|
|
30
|
+
// what let a stale dispatcher pass for a successful upgrade.
|
|
31
|
+
expect(result.success).toBe(false);
|
|
32
|
+
if (!result.success) {
|
|
33
|
+
expect(result.error).toContain('restart the event dispatcher');
|
|
34
|
+
expect(result.error).toContain('apt-get upgrade');
|
|
35
|
+
expect(result.error).toContain('apply DB migrations');
|
|
36
|
+
}
|
|
37
|
+
});
|
|
38
|
+
|
|
20
39
|
test('stops at the first failing step and does not run later ones', async () => {
|
|
21
40
|
const seen: string[][] = [];
|
|
22
41
|
const result = await handleAptUpgrade([], {}, (argv) => {
|
|
@@ -34,6 +34,11 @@ const STEPS: Step[] = [
|
|
|
34
34
|
argv: ['sudo', 'apt-get', '-y', '--only-upgrade', 'install', 'celilo', 'celilo-bootstrap'],
|
|
35
35
|
},
|
|
36
36
|
{ label: 'apply DB migrations', argv: [CELILO_BIN, 'system', 'migrate'] },
|
|
37
|
+
// The dispatcher runs the code it LOADED, not the code on disk. Without this
|
|
38
|
+
// step celilo-mgr sat 9 days on event-bus v0.1.8 after apt installed v0.2.0,
|
|
39
|
+
// faithfully running the bug the upgrade shipped to fix (celilo#604). The
|
|
40
|
+
// upgraded binary does the restart so the verification is the new code's.
|
|
41
|
+
{ label: 'restart the event dispatcher', argv: [CELILO_BIN, 'events', 'restart-daemon'] },
|
|
37
42
|
];
|
|
38
43
|
|
|
39
44
|
/** Run one argv, inheriting stdio so its output streams through api-serve. */
|
|
@@ -46,18 +51,23 @@ export async function handleAptUpgrade(
|
|
|
46
51
|
_flags: Record<string, string | boolean>,
|
|
47
52
|
runStep: StepRunner = defaultRunner,
|
|
48
53
|
): Promise<CommandResult> {
|
|
54
|
+
const done: string[] = [];
|
|
49
55
|
for (const step of STEPS) {
|
|
50
56
|
process.stdout.write(`\n▸ ${step.label}\n`);
|
|
51
57
|
const { status } = runStep(step.argv);
|
|
52
58
|
if (status !== 0) {
|
|
59
|
+
// Name what DID happen. A failure on the last step means new code is
|
|
60
|
+
// installed and the dispatcher is still serving the old — the operator
|
|
61
|
+
// has to be told that in the failure itself, not left to infer it.
|
|
53
62
|
return {
|
|
54
63
|
success: false,
|
|
55
|
-
error: `apt-upgrade failed at "${step.label}" (exit ${status ?? 'signal'}). Nothing further was run.`,
|
|
64
|
+
error: `apt-upgrade failed at "${step.label}" (exit ${status ?? 'signal'}). Completed: ${done.length > 0 ? done.join(', ') : 'nothing'}. Nothing further was run.`,
|
|
56
65
|
};
|
|
57
66
|
}
|
|
67
|
+
done.push(step.label);
|
|
58
68
|
}
|
|
59
69
|
return {
|
|
60
70
|
success: true,
|
|
61
|
-
message: 'celilo apt packages upgraded
|
|
71
|
+
message: 'celilo apt packages upgraded, migrations applied, dispatcher restarted on new code.',
|
|
62
72
|
};
|
|
63
73
|
}
|
|
@@ -9,17 +9,68 @@
|
|
|
9
9
|
* report counts.
|
|
10
10
|
*/
|
|
11
11
|
|
|
12
|
+
import { readdirSync, rmSync } from 'node:fs';
|
|
13
|
+
import { tmpdir } from 'node:os';
|
|
14
|
+
import { join } from 'node:path';
|
|
12
15
|
import {
|
|
13
16
|
createModuleBackup,
|
|
14
17
|
findBackupEligibleModules,
|
|
15
18
|
isBackupDue,
|
|
16
19
|
} from '../../services/backup-create';
|
|
20
|
+
import { failBackup, getBackup } from '../../services/backup-metadata';
|
|
17
21
|
import { pruneBackupsForModule } from '../../services/backup-retention';
|
|
22
|
+
import {
|
|
23
|
+
ABANDONED_BACKUP_MESSAGE,
|
|
24
|
+
STAGING_PREFIX,
|
|
25
|
+
impliesAbandonedRecord,
|
|
26
|
+
reapOrphanedStaging,
|
|
27
|
+
} from '../../services/backup-staging';
|
|
18
28
|
import { type BackupSweepReport, runBackupSweep } from '../../services/backup-sweep';
|
|
29
|
+
import { isPidRunnable } from '../../services/module-operations';
|
|
19
30
|
import type { CommandResult } from '../types';
|
|
20
31
|
|
|
32
|
+
/**
|
|
33
|
+
* Reclaim orphaned staging, then correct the records that were still claiming
|
|
34
|
+
* to be running.
|
|
35
|
+
*
|
|
36
|
+
* The record fix-up lives here rather than inside the reaper because the reaper
|
|
37
|
+
* is the filesystem decision and this is a database write; both follow from the
|
|
38
|
+
* one liveness lookup, so neither repeats it.
|
|
39
|
+
*/
|
|
40
|
+
function reapStaging() {
|
|
41
|
+
const report = reapOrphanedStaging({
|
|
42
|
+
listStagingDirs: () => {
|
|
43
|
+
// A missing or unreadable temp dir is not a reason to fail the sweep.
|
|
44
|
+
try {
|
|
45
|
+
return readdirSync(tmpdir())
|
|
46
|
+
.filter((name) => name.startsWith(STAGING_PREFIX))
|
|
47
|
+
.map((name) => join(tmpdir(), name));
|
|
48
|
+
} catch {
|
|
49
|
+
return [];
|
|
50
|
+
}
|
|
51
|
+
},
|
|
52
|
+
lookupOwner: (recordId) => {
|
|
53
|
+
const record = getBackup(recordId);
|
|
54
|
+
if (!record) return null;
|
|
55
|
+
return { status: record.status, pid: record.pid, startedAt: record.startedAt };
|
|
56
|
+
},
|
|
57
|
+
isPidRunnable,
|
|
58
|
+
remove: (path) => rmSync(path, { recursive: true, force: true }),
|
|
59
|
+
now: () => Date.now(),
|
|
60
|
+
});
|
|
61
|
+
|
|
62
|
+
for (const reclaimed of report.reclaimed) {
|
|
63
|
+
if (impliesAbandonedRecord(reclaimed.reason)) {
|
|
64
|
+
failBackup(reclaimed.recordId, ABANDONED_BACKUP_MESSAGE);
|
|
65
|
+
}
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
return report;
|
|
69
|
+
}
|
|
70
|
+
|
|
21
71
|
export async function handleBackupSweep(): Promise<CommandResult> {
|
|
22
72
|
const report = await runBackupSweep({
|
|
73
|
+
reapStaging,
|
|
23
74
|
listEligible: () =>
|
|
24
75
|
findBackupEligibleModules().map(({ module, manifest }) => ({ id: module.id, manifest })),
|
|
25
76
|
isDue: (moduleId, schedule) => isBackupDue(moduleId, schedule),
|
|
@@ -55,6 +106,17 @@ function formatReport(report: BackupSweepReport): string {
|
|
|
55
106
|
if (report.failures.length > 0) parts.push(`${report.failures.length} FAILED`);
|
|
56
107
|
|
|
57
108
|
const lines = [`backup sweep: ${parts.join(', ')}`];
|
|
109
|
+
// Reclamation is reported even though it is housekeeping: it is the only
|
|
110
|
+
// visible evidence that backups have been dying, and a silent reaper would
|
|
111
|
+
// hide the very failure it exists to clean up after.
|
|
112
|
+
if (report.staging.reclaimed.length > 0) {
|
|
113
|
+
lines.push(
|
|
114
|
+
` reclaimed ${report.staging.reclaimed.length} orphaned staging dir(s) from backups that were killed before cleanup`,
|
|
115
|
+
);
|
|
116
|
+
for (const reclaimed of report.staging.reclaimed) {
|
|
117
|
+
lines.push(` ${reclaimed.recordId} (${reclaimed.reason})`);
|
|
118
|
+
}
|
|
119
|
+
}
|
|
58
120
|
for (const moduleId of report.skippedLocked) {
|
|
59
121
|
lines.push(` skipped ${moduleId}: another operation is in flight — retrying next tick`);
|
|
60
122
|
}
|
|
@@ -20,7 +20,9 @@
|
|
|
20
20
|
* resume alias for repair (acknowledges halt-on-recovery)
|
|
21
21
|
*/
|
|
22
22
|
|
|
23
|
+
import { spawnSync } from 'node:child_process';
|
|
23
24
|
import {
|
|
25
|
+
BUS_VERSION,
|
|
24
26
|
defineEvents,
|
|
25
27
|
drainOnce,
|
|
26
28
|
openBus,
|
|
@@ -37,10 +39,15 @@ import type { HookName } from '../../hooks/types';
|
|
|
37
39
|
import type { ModuleManifest } from '../../manifest/schema';
|
|
38
40
|
import type { EnsureRequiredPayload } from '../../services/bus-interview';
|
|
39
41
|
import {
|
|
42
|
+
detectPlatform,
|
|
40
43
|
installDaemon,
|
|
41
44
|
planDaemonInstall,
|
|
42
45
|
readInstalledUnit,
|
|
46
|
+
resolveRestartScope,
|
|
47
|
+
restartDaemon,
|
|
48
|
+
supervisorCommands,
|
|
43
49
|
uninstallDaemon,
|
|
50
|
+
unitInstalledInAnyScope,
|
|
44
51
|
} from '../../services/events-daemon';
|
|
45
52
|
import { getArg, hasFlag } from '../parser';
|
|
46
53
|
import type { CommandResult } from '../types';
|
|
@@ -897,6 +904,89 @@ export async function handleEventsUninstallDaemon(
|
|
|
897
904
|
}
|
|
898
905
|
}
|
|
899
906
|
|
|
907
|
+
/**
|
|
908
|
+
* `celilo events restart-daemon` — cycle the dispatcher through its supervisor
|
|
909
|
+
* and PROVE the new process is live on the installed code (celilo#604).
|
|
910
|
+
*
|
|
911
|
+
* The interesting case is the orphan: a dispatcher the supervisor doesn't own
|
|
912
|
+
* can't be killed by `systemctl restart`, and the one-dispatcher-per-bus guard
|
|
913
|
+
* (#584) then crash-loops the unit while the old code keeps serving. So this
|
|
914
|
+
* stops unmanaged dispatchers first, and verifies on the bus afterwards rather
|
|
915
|
+
* than trusting systemctl's exit code.
|
|
916
|
+
*/
|
|
917
|
+
export async function handleEventsRestartDaemon(
|
|
918
|
+
_args: string[],
|
|
919
|
+
flags: Record<string, string | boolean>,
|
|
920
|
+
): Promise<CommandResult> {
|
|
921
|
+
const bus = openCliBus();
|
|
922
|
+
try {
|
|
923
|
+
const platform = detectPlatform();
|
|
924
|
+
// No unit installed at all. apt-upgrade runs this on every box, including
|
|
925
|
+
// ones that never installed the daemon — failing those would break an
|
|
926
|
+
// upgrade that has nothing stale to fix. Distinguish the two cases:
|
|
927
|
+
// nothing running is a genuine no-op; something running is unsupervised
|
|
928
|
+
// and may be stale, and there is no supervisor to cycle it through.
|
|
929
|
+
if (!flags.system && !unitInstalledInAnyScope(platform)) {
|
|
930
|
+
const live = bus.liveDispatchers();
|
|
931
|
+
if (live.length === 0) {
|
|
932
|
+
return {
|
|
933
|
+
success: true,
|
|
934
|
+
message: 'No supervisor unit installed and no dispatcher running — nothing to restart.',
|
|
935
|
+
};
|
|
936
|
+
}
|
|
937
|
+
return {
|
|
938
|
+
success: false,
|
|
939
|
+
error: `A dispatcher is running unsupervised (pid ${live.map((d) => d.pid).join(', ')}, code v${live[0]?.version ?? '?'}) and no supervisor unit is installed, so it cannot be cycled — it may be serving stale code. Run \`celilo events install-daemon\`, enable the unit, then retry.`,
|
|
940
|
+
};
|
|
941
|
+
}
|
|
942
|
+
const scope = resolveRestartScope({ platform, scope: flags.system ? 'system' : undefined });
|
|
943
|
+
const cmds = supervisorCommands(platform, scope);
|
|
944
|
+
const result = await restartDaemon(
|
|
945
|
+
{ platform, scope, expectedVersion: BUS_VERSION },
|
|
946
|
+
{
|
|
947
|
+
liveDispatchers: () =>
|
|
948
|
+
bus.liveDispatchers().map((d) => ({ pid: d.pid, version: d.version })),
|
|
949
|
+
supervisorPid: () => {
|
|
950
|
+
if (!cmds.mainPid) return null;
|
|
951
|
+
const out = spawnSync(cmds.mainPid[0], cmds.mainPid.slice(1), { encoding: 'utf-8' });
|
|
952
|
+
const pid = Number((out.stdout ?? '').trim());
|
|
953
|
+
return Number.isInteger(pid) && pid > 0 ? pid : null;
|
|
954
|
+
},
|
|
955
|
+
kill: (pid, signal) => {
|
|
956
|
+
try {
|
|
957
|
+
process.kill(pid, signal);
|
|
958
|
+
} catch {
|
|
959
|
+
// Already gone, or not ours to signal — the bus poll is the arbiter.
|
|
960
|
+
}
|
|
961
|
+
},
|
|
962
|
+
restartUnit: () => {
|
|
963
|
+
const out = spawnSync(cmds.restart[0], cmds.restart.slice(1), { encoding: 'utf-8' });
|
|
964
|
+
if (out.status !== 0) {
|
|
965
|
+
throw new Error(
|
|
966
|
+
`${cmds.restart.join(' ')} failed (exit ${out.status ?? 'signal'}): ${(out.stderr ?? '').trim()}`,
|
|
967
|
+
);
|
|
968
|
+
}
|
|
969
|
+
},
|
|
970
|
+
sleep: (ms) => new Promise((r) => setTimeout(r, ms)),
|
|
971
|
+
},
|
|
972
|
+
);
|
|
973
|
+
const lines = [
|
|
974
|
+
`Dispatcher restarted under the ${result.scope}-scope unit: ${shortenPath(result.unitPath)}`,
|
|
975
|
+
` now running: pid ${result.dispatcher.pid}, code v${result.dispatcher.version}`,
|
|
976
|
+
...(result.orphansKilled.length > 0
|
|
977
|
+
? [
|
|
978
|
+
` stopped ${result.orphansKilled.length} unsupervised dispatcher(s): pid ${result.orphansKilled.join(', ')}`,
|
|
979
|
+
]
|
|
980
|
+
: []),
|
|
981
|
+
];
|
|
982
|
+
return { success: true, message: lines.join('\n'), data: result };
|
|
983
|
+
} catch (err) {
|
|
984
|
+
return { success: false, error: err instanceof Error ? err.message : String(err) };
|
|
985
|
+
} finally {
|
|
986
|
+
bus.close();
|
|
987
|
+
}
|
|
988
|
+
}
|
|
989
|
+
|
|
900
990
|
/**
|
|
901
991
|
* `celilo events show-daemon` — print whatever unit file is currently
|
|
902
992
|
* installed (`--system` for the system-scope unit) so the operator can
|