@celilo/cli 0.14.2 → 0.15.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CELILO_CORE_MODULES.md +1 -1
- package/CELILO_SUBSYSTEMS.md +18 -2
- package/drizzle/0018_drop_alert_policy_snapshot.sql +46 -0
- package/drizzle/meta/_journal.json +8 -1
- package/package.json +4 -4
- package/src/capabilities/public-web-publish.test.ts +15 -15
- package/src/cli/commands/alerts-list.ts +10 -0
- package/src/cli/commands/alerts-poll.ts +12 -6
- package/src/cli/commands/alerts-sweep.ts +22 -81
- package/src/cli/commands/backup-sweep.ts +65 -0
- package/src/cli/commands/module-operations.test.ts +93 -0
- package/src/cli/commands/module-operations.ts +134 -0
- package/src/cli/commands/module-upgrade.test.ts +32 -20
- package/src/cli/commands/module-upgrade.ts +37 -32
- package/src/cli/commands/monitor.ts +26 -6
- package/src/cli/commands/system-audit.ts +9 -68
- package/src/cli/completion.ts +18 -1
- package/src/cli/index.ts +11 -0
- package/src/db/schema.ts +5 -3
- package/src/hooks/capability-loader.ts +0 -12
- package/src/manifest/schema.ts +4 -1
- package/src/module/packaging/build.ts +4 -0
- package/src/services/alerting/builtin-source.ts +18 -51
- package/src/services/alerting/delivery-loop.test.ts +5 -1
- package/src/services/alerting/format.test.ts +0 -1
- package/src/services/alerting/inbound-poller.test.ts +44 -8
- package/src/services/alerting/inbound-poller.ts +65 -28
- package/src/services/alerting/notify-deps.ts +113 -0
- package/src/services/alerting/run-monitor.ts +0 -1
- package/src/services/alerting/store.test.ts +1 -1
- package/src/services/alerting/store.ts +0 -2
- package/src/services/alerting/sweep-runner.test.ts +11 -2
- package/src/services/alerting/sweep-runner.ts +14 -7
- package/src/services/audit/backup-source.ts +54 -0
- package/src/services/audit/backups.test.ts +7 -2
- package/src/services/audit/backups.ts +10 -18
- package/src/services/backup-cipher.test.ts +188 -0
- package/src/services/backup-cipher.ts +178 -0
- package/src/services/backup-create.ts +20 -30
- package/src/services/backup-envelope-roundtrip.test.ts +6 -26
- package/src/services/backup-restore.ts +10 -16
- package/src/services/backup-schedule.ts +35 -0
- package/src/services/backup-sweep.test.ts +148 -0
- package/src/services/backup-sweep.ts +124 -0
- package/src/services/deploy-posture.ts +15 -2
- package/src/services/machine-probe.test.ts +50 -0
- package/src/services/machine-probe.ts +73 -0
- package/src/services/module-operations.test.ts +67 -6
- package/src/services/module-operations.ts +69 -19
- package/src/services/module-subscriptions.test.ts +33 -2
- package/src/services/module-subscriptions.ts +10 -1
- package/src/services/module-validator/typescript-build.test.ts +20 -1
- package/src/services/module-validator/typescript-build.ts +9 -5
- package/src/services/restore-from-file.ts +6 -21
- package/src/templates/generator.test.ts +88 -0
- package/src/templates/generator.ts +119 -16
package/CELILO_CORE_MODULES.md
CHANGED
|
@@ -39,7 +39,7 @@ Each entry: `module id` — what it is — **provides** / **requires** capabilit
|
|
|
39
39
|
- **celilo-mgmt** — the celilo management server itself, deployed as a module (replaces install.sh + `system init`; ships daemon, runs migrations, self-registers). **provides:** `celilo_event_bus`, `celilo_module_deploy_worker`. **requires:** `cross_module_read`. See `openspec/specs/management-as-module/spec.md`.
|
|
40
40
|
- **celilo-registry** — module registry server (Cargo sparse protocol); stores `.netapp` files, serves index + search/download API. On install it provisions a confidential introspection OIDC client via `idp.create_oidc_client` (SECURE_MODULE_PUBLISH.md §5[D-A]) and converges its issuer + introspection endpoint + creds onto the box for RFC 7662 token verification. **provides:** `registry_publish`. **requires:** `public_web`, `dns_registrar`, `idp`.
|
|
41
41
|
- **celilo-apt-repo** — Debian apt repository (reprepro + Bun HTTP server) serving the celilo `.deb` at apt.celilo.computer. **provides:** `apt_publish`. **requires:** `public_web`, `dns_registrar`.
|
|
42
|
-
- **signal** — bidirectional Signal transport for alerts and deploy-interview questions; runs signal-cli in daemon mode with its JSON-RPC socket bound to the host's own address (never public). Enrolled as a SECONDARY DEVICE of an existing Signal account rather than registering its own number — Signal blocks most VOIP ranges and bans bot-ish registrations. Recipient addresses live on celilo routes, not in module config, so adding a person never requires a redeploy. Runs on x86_64 and aarch64. `libsignal-client` ships no linux-aarch64 native, so celilo builds one (`modules/signal/build/`) and installs it as a `libsignal-jni` .deb on ARM hosts; x86_64 uses the JAR's bundled native. **provides:** `notification` (`send`, `receive`). **requires:** no capabilities — a transport that depended on the proxy, registrar or firewall could not tell you those were broken — and a system in the **`secure-mgmt`** zone: it holds a linked Signal account (the operator's own messaging identity and keys), and its job is to observe every tier while depending on none, which is what the control-plane zone is for. See `openspec/changes/add-alerting/`.
|
|
42
|
+
- **signal** — bidirectional Signal transport for alerts and deploy-interview questions; runs signal-cli in daemon mode with its JSON-RPC socket bound to the host's own address (never public) and **`--receive-mode=manual`**, which is load-bearing: signal-cli's default (`on-start`) leaves the daemon permanently receiving, so it drains every reply into an SSE stream nothing is attached to and REFUSES celilo's `receive` call — replies arrive and are unreadable, while `send` and every health check keep passing. Enrolled as a SECONDARY DEVICE of an existing Signal account rather than registering its own number — Signal blocks most VOIP ranges and bans bot-ish registrations. Recipient addresses live on celilo routes, not in module config, so adding a person never requires a redeploy. Runs on x86_64 and aarch64. `libsignal-client` ships no linux-aarch64 native, so celilo builds one (`modules/signal/build/`) and installs it as a `libsignal-jni` .deb on ARM hosts; x86_64 uses the JAR's bundled native. **provides:** `notification` (`send`, `receive`). **requires:** no capabilities — a transport that depended on the proxy, registrar or firewall could not tell you those were broken — and a system in the **`secure-mgmt`** zone: it holds a linked Signal account (the operator's own messaging identity and keys), and its job is to observe every tier while depending on none, which is what the control-plane zone is for. See `openspec/changes/add-alerting/`.
|
|
43
43
|
- **celilo-website** — public docs site (static Astro) served via Caddy on celilo.computer. **requires:** `public_web`, `dns_registrar`.
|
|
44
44
|
|
|
45
45
|
## Git forge & CI pipeline
|
package/CELILO_SUBSYSTEMS.md
CHANGED
|
@@ -143,6 +143,8 @@ runner seam (`execRunner` real / `createMockRunner` for tests) lives in
|
|
|
143
143
|
- **Deploy pipeline** — `apps/celilo/src/services/module-deploy.ts`.
|
|
144
144
|
- **DNS provider backfill** (re-emit registrations when a provider deploys) — `apps/celilo/src/services/dns-provider-backfill.ts` — `isDnsInternalProvider`, `backfillProviderDns`.
|
|
145
145
|
- **Base-module aspects (fan-out across the fleet)** — `apps/celilo/src/services/aspect-runner.ts` — `planAspectFanOut`, `runAspectFanOut`, `maybeRunAspectForTrigger`. Aspect content lives in `modules/<m>/base-module-aspect/` (e.g. knot-unbound-internal, technitium).
|
|
146
|
+
- **In-flight operation lock** — `apps/celilo/src/services/module-operations.ts` — `startOperation`/`completeOperation`/`failOperation` record deploy/uninstall/backup/restore in `module_operations`; `refuseIfInFlight`/`checkInFlight` are what backup and restore consult. Deploy and uninstall REGISTER but never check: it is a one-way guard protecting backup/restore consistency, not a general mutex (`openspec/specs/management-server-backup/spec.md` "In-flight operation refusal"). A row stops holding the lock once it is GONE, STOPPED/zombie (`isPidRunnable`, `ps -o state=` — `kill(pid,0)` calls a Ctrl-Z'd process alive), or older than `OPERATION_TTL_MS` (2h). The TTL is not redundancy: a pid is a recycled number, and once the pid space wraps an old row names an unrelated healthy process. Operator surface: `celilo module operations [list|clear] [--all]` (`apps/celilo/src/cli/commands/module-operations.ts`).
|
|
147
|
+
- **Backup artifact encryption** — `apps/celilo/src/services/backup-cipher.ts` — `encryptFileToFile`/`decryptFileToFile`, file-in/file-out and streamed, used by every backup writer (`backup-create.ts`) and reader (`backup-restore.ts`, `restore-from-file.ts`). Do NOT route artifacts through `secrets/encryption.ts`: that API is string-in/string-out for short DB values, and feeding it a tar cost base64 (1.33x) then hex (2x) then `JSON.stringify` — ~9x the artifact in memory, which OOM-killed forgejo's 774 MB backup, and a hard ~805 MB ceiling from the max string length that no amount of RAM raises. On-disk format is `magic "CELILOBK" (8) | version (1) | iv (16) | ciphertext | GCM tag (16)`; the tag is a trailer because it does not exist until the last byte is encrypted. `decryptFileToFile` still reads the pre-2026-07 JSON-envelope artifacts, discriminating on the magic bytes — the envelope's own `schemaVersion` cannot serve, as it lives inside the encrypted tar.
|
|
146
148
|
|
|
147
149
|
## Generation & templating
|
|
148
150
|
|
|
@@ -171,6 +173,7 @@ is currently wrong, and routes carry the message to a person's phone. Design:
|
|
|
171
173
|
- **Alert identity** — `apps/celilo/src/services/alerting/keys.ts` — the key grammar (`module:<id>[/check:<name>]`, `builtin:<check>[/<kind>:<target>]`) that makes "the same problem" the same alert across runs. `moduleAlertKey`, `moduleCheckAlertKey`, `builtinAlertKey`, `parseAlertKey`.
|
|
172
174
|
- **Reconciliation** — `apps/celilo/src/services/alerting/reconcile.ts` (`reconcile`) — a successful run's failing-key set is authoritative and resolution is by SET DIFFERENCE (absent ⇒ resolved). A run whose outcome is `error` resolves NOTHING and fires a module-level alert instead: the false-all-clear guard.
|
|
173
175
|
- **Monitor execution** — `apps/celilo/src/services/alerting/run-monitor.ts` (`runOneMonitor`) + `sweep.ts` (`selectDueMonitors`) + `builtin-monitors.ts` / `health-coverage.ts` (the built-in checks and the "module with no health check" coverage check).
|
|
176
|
+
- **Scheduled audit categories (`builtin_check` monitors)** — `apps/celilo/src/services/alerting/builtin-source.ts` — `SCHEDULABLE_BUILTIN_CHECKS` is the list of `celilo system audit` categories cheap enough to run every sweep: `machines_reachable` and `backups`. Everything else in the audit needs the whole world injected (proxmox, terraform, registry) and is not schedulable. Enable one with `celilo monitor add backups --interval 1h`. The `backups` roster comes from `apps/celilo/src/services/audit/backup-source.ts` (`loadBackupAuditInfo`), shared with `celilo system audit` so both judge the same fleet.
|
|
174
177
|
- **The sweep** — `apps/celilo/src/services/alerting/sweep-runner.ts` (`runSweep`) — the ordered pass that makes alerting run by itself: run due monitors → promote past-grace alerts → re-evaluate suppression → flush quiet-hours deferrals → notify. Driven by `celilo alerts sweep` on `timer.tick.5m`. Never throws for one bad monitor.
|
|
175
178
|
- **Suppression (topology-derived, never configured)** — `apps/celilo/src/services/alerting/suppression.ts` — `ancestorKeysFor`/`findSuppressor`/`machineAlertKey`. A firing machine explains its modules' failures; a firing capability provider explains its zone's consumers. Derived from `module_systems`, so it cannot drift from reality. Deploy windows: `deploy-hooks.ts` (`openDeployWindow`/`closeDeployWindows` — closed by module, so a crashed deploy self-heals).
|
|
176
179
|
- **Escalation & quiet hours** — `escalation.ts` (`decideEscalation`, every reason to stay silent enumerated) + `quiet-hours.ts` (`isWithinQuietHours`, Intl-based and DST-safe). Quiet hours defer the MESSAGE while the escalation clock keeps running.
|
|
@@ -179,9 +182,21 @@ is currently wrong, and routes carry the message to a person's phone. Design:
|
|
|
179
182
|
- **Ack / silence / resolve** — `ack.ts` — three deliberately distinct operations. An ack is broadcast to every other paged route.
|
|
180
183
|
- **Interviews over a transport** — `interview-responder.ts` (delivery policy; `secret.*` families are REFUSED) + `notification-responder.ts` (queries the bus's events table for unanswered questions: a watch only sees events emitted after it registers, and `alerts poll` is one-shot. Leaves a question alone for 90s so an attached responder — terminal, or the `api-serve` wire bridge — answers first). Alerts and interviews share the send path, the token table, and the inbound path; only `targetId` differs.
|
|
181
184
|
- **People, routes, policies** — `people.ts` — `createPerson`/`createRoute`/`createPolicy`/`addPolicyStep`.
|
|
182
|
-
- **
|
|
185
|
+
- **Policy resolution** — `apps/celilo/src/services/alerting/notify-deps.ts` — `policyForAlert` (the single answer to "who would this page") + `buildNotifyDeps` (assembles steps, routes, quiet hours, transport, reply token). The policy is read from the MONITOR on every sweep, never snapshotted onto the alert, so `escalation-policy assign` takes effect on alerts that are already firing.
|
|
186
|
+
- **Persistence** — `apps/celilo/src/services/alerting/store.ts`; tables `people`, `routes`, `escalation_policies`, `escalation_steps`, `monitors`, `monitor_runs`, `suppression_windows`, `alerts`, `notification_deliveries` (`apps/celilo/src/db/schema.ts`, migrations `drizzle/0017_alerting.sql`, `drizzle/0018_drop_alert_policy_snapshot.sql`).
|
|
183
187
|
- **CLI** — `apps/celilo/src/cli/commands/` — `alerts-list.ts`, `alerts-act.ts` (ack/silence/resolve), `alerts-sweep.ts`, `alerts-poll.ts`, `monitor.ts`, `notify-config.ts` (`person`/`route`/`escalation-policy`).
|
|
184
188
|
|
|
189
|
+
## Backups
|
|
190
|
+
|
|
191
|
+
Creation, scheduling and freshness. A module declares an `on_backup` hook and a
|
|
192
|
+
`backup.schedule`; celilo runs it on that cadence and alerts when it stops.
|
|
193
|
+
|
|
194
|
+
- **Creation** — `apps/celilo/src/services/backup-create.ts` — `createModuleBackup` (invokes the module's `on_backup` hook into an encrypted envelope), `createSystemStateBackup` (celilo.db), `findBackupEligibleModules`, `isBackupDue`. Storage destinations: `backup-storage.ts`. Retention: `backup-retention.ts` (`pruneBackupsForModule`). Restore: `backup-restore.ts`.
|
|
195
|
+
- **Cadence (one accessor)** — `apps/celilo/src/services/backup-schedule.ts` — `effectiveBackupSchedule(manifest)`. An absent `backup.schedule` means `daily`, NOT `manual`; opting out takes an explicit `manual`. Both the freshness audit and the backup sweep must read cadence through this one function, or a module can be alerted-on but never backed up.
|
|
196
|
+
- **The sweep** — `apps/celilo/src/services/backup-sweep.ts` — `runBackupSweep` (the pass that makes backups run by themselves: for each eligible module, is its declared cadence due → back it up → apply declared retention) + `ensureBackupSweepSubscriber`. Driven by `celilo backup sweep` on `timer.tick.1h` — the coarsest tick that can still serve an `hourly` cadence. Armed automatically: `registerModuleSubscriptions` registers the subscriber for any module declaring an `on_backup` hook, so it appears on install or `module update`. A run refused by the in-flight operation lock is a skip retried next tick, never a failure.
|
|
197
|
+
- **Freshness audit** — `apps/celilo/src/services/audit/backups.ts` (`auditBackups`) — `backup_missing` / `backup_stale` drift findings against the same declared cadence.
|
|
198
|
+
- **CLI** — `apps/celilo/src/cli/commands/` — `backup-sweep.ts`, `backup-create.ts` (also `celilo module backup`), `backup-list.ts`, `backup-restore.ts`, `backup-prune.ts`, `backup-delete.ts`, `backup-import.ts`, `backup-pull.ts`, `backup-name.ts`.
|
|
199
|
+
|
|
185
200
|
## Persistence
|
|
186
201
|
|
|
187
202
|
- **DB schema** — `apps/celilo/src/db/schema.ts`. Client: `apps/celilo/src/db/client.ts`. Migration runner: `apps/celilo/src/db/migrate.ts`. Migrations: `apps/celilo/drizzle/`.
|
|
@@ -209,5 +224,6 @@ Run any celilo command on celilo-mgr over SSH instead of screen-scraping `ssh <h
|
|
|
209
224
|
|
|
210
225
|
- **cele2e harness** — `packages/e2e/src/` — `runner.ts`, `container-manager.ts` (`startNetwork`, `reconnectNetwork`), `network-builder.ts` (`NetworkBuilder`).
|
|
211
226
|
- **Run wrapper** — `infra/scripts/cele2e-run.sh` lives in the separate `infra/` clone, **not** in this repo. See the cele2e section of the repo-root `CLAUDE.md` for the operator workflow.
|
|
212
|
-
- **Signal simulators** — three containers, three jobs. `docker/Dockerfile.signal-cli` runs the REAL unlinked daemon (`network().withSignalCli()`, reachable at `signal-cli.lab`) so `e2e/tests/signal-contract.test.ts` can re-check celilo's understanding of the JSON-RPC surface against the actual binary — the pebble pattern. `docker/Dockerfile.signal-sim` runs `simulators/signal-cli/server.ts` (`withSignalSim()`, `signal-sim.lab`), the drivable stand-in with a control surface (`/_control/inbound`, `/_control/sent`, `/_control/unlink`). `docker/Dockerfile.signal-release` (`withSignalRelease()`, `signal-release.lab`) serves the signal-cli release tarball so the module's deploy-time download resolves inside the sealed network — the download is served, never skipped. The libsignal aarch64 native is compiled at `build-infra` time from the module's own recipe (`packages/e2e/scripts/stage-libsignal.ts` → `modules/signal/build/`) and staged into the apt-repo pool, so the recipe is exercised for real on every rebuild while the deploy stays fast and the network stays sealed.
|
|
227
|
+
- **Signal simulators** — three containers, three jobs. `docker/Dockerfile.signal-cli` runs the REAL unlinked daemon (`network().withSignalCli()`, reachable at `signal-cli.lab`) so `e2e/tests/signal-contract.test.ts` can re-check celilo's understanding of the JSON-RPC surface against the actual binary — the pebble pattern. `docker/Dockerfile.signal-sim` runs `simulators/signal-cli/server.ts` (`withSignalSim()`, `signal-sim.lab`), the drivable stand-in with a control surface (`/_control/inbound`, `/_control/sent`, `/_control/unlink`). `/_control/inbound` from the LINKED account's own number emits a `syncMessage.sentMessage` transcript rather than a `dataMessage`, because that is the only shape the real daemon delivers a note-to-self in — and a note-to-self is what every reply is in the default single-operator setup (#460). `docker/Dockerfile.signal-release` (`withSignalRelease()`, `signal-release.lab`) serves the signal-cli release tarball so the module's deploy-time download resolves inside the sealed network — the download is served, never skipped. The libsignal aarch64 native is compiled at `build-infra` time from the module's own recipe (`packages/e2e/scripts/stage-libsignal.ts` → `modules/signal/build/`) and staged into the apt-repo pool, so the recipe is exercised for real on every rebuild while the deploy stays fast and the network stays sealed.
|
|
228
|
+
- **Public-boundary NAT model** — `packages/e2e/config/routing/` — exactly ONE NAT sits between the fleet and the simulated internet: the customer firewall (`fw-main` in `direct-internet`, `fw-isp` in the two-layer default), which MASQUERADEs to `100.100.0.100`. The ISP edge `fw-ext` ROUTES the customer's `100.100.0.0/24` and must never re-NAT it (`-s 100.100.0.0/24 -j RETURN` ahead of its MASQUERADE) — an ISP does not NAT a subscriber that already holds a public address. This is load-bearing, not cosmetic: Namecheap-style DDNS registers the SOURCE address when the caller omits `ip=`, which is how celilo registers public names since #464/#466, so a second NAT here publishes the simulator's own address for every public hostname and quietly breaks ACME, inbound reach, and seeded apex records. Constants: `externalWanIp()` / `externalWanSubnet()` in `src/types.ts`. The corollary: every simulator on `internet-external` must default-route via fw-ext (`100.64.0.1`) so it can reply to the customer's public address — Docker's bridge gateway has no path across networks. `config/routing/public-sim-entrypoint.sh` is the shared two-liner; the DNS hierarchy, pebble, isitup and celilo-website-sim already carried it, and npm-registry / registry / apt-repo / minio / cpanel-host only appeared to work because the double NAT put fw-ext's on-link address in the source field.
|
|
213
229
|
- **MCP server (agent-driven e2e)** — `packages/mcp-server/src/index.ts` — stdio MCP server exposing `start_run`/`run_status`/`run_result`/`stop_run` (detached cele2e runs read off the event bus, no ANSI scraping) + `env_check` (docker VM, run-lock, shared-infra, mgmt image CLI version, netapps). Dev/ops tool, `private`, not shipped to consumers.
|
|
@@ -0,0 +1,46 @@
|
|
|
1
|
+
-- Drop `alerts.escalation_policy_id`.
|
|
2
|
+
--
|
|
3
|
+
-- The escalation policy was copied onto each alert when the alert was created,
|
|
4
|
+
-- and the sweep resolved it from that copy. An operator assigning a policy to a
|
|
5
|
+
-- monitor therefore changed nothing for any alert that was ALREADY firing —
|
|
6
|
+
-- which is the only situation in which anyone runs `escalation-policy assign`.
|
|
7
|
+
-- The policy now lives only on the monitor and is resolved on every sweep.
|
|
8
|
+
--
|
|
9
|
+
-- Table rebuild rather than DROP COLUMN: SQLite refuses to drop a column named
|
|
10
|
+
-- in a FOREIGN KEY clause.
|
|
11
|
+
|
|
12
|
+
CREATE TABLE `__new_alerts` (
|
|
13
|
+
`id` text PRIMARY KEY NOT NULL,
|
|
14
|
+
`key` text NOT NULL,
|
|
15
|
+
`active_key` text,
|
|
16
|
+
`monitor_id` text NOT NULL,
|
|
17
|
+
`state` text DEFAULT 'pending' NOT NULL,
|
|
18
|
+
`severity` text NOT NULL,
|
|
19
|
+
`first_fired_at` integer DEFAULT (unixepoch()) NOT NULL,
|
|
20
|
+
`last_seen_at` integer DEFAULT (unixepoch()) NOT NULL,
|
|
21
|
+
`grace_until` integer NOT NULL,
|
|
22
|
+
`suppressed_by_alert_id` text,
|
|
23
|
+
`suppressed_by_window_id` text,
|
|
24
|
+
`unsuppressed_at` integer,
|
|
25
|
+
`awaiting_confirmation` integer DEFAULT false NOT NULL,
|
|
26
|
+
`acked_by` text,
|
|
27
|
+
`acked_at` integer,
|
|
28
|
+
`silenced_until` integer,
|
|
29
|
+
`escalation_step` integer DEFAULT 0 NOT NULL,
|
|
30
|
+
`next_escalation_at` integer,
|
|
31
|
+
`deferred_until` integer,
|
|
32
|
+
`deferred_route_id` text,
|
|
33
|
+
`message` text NOT NULL,
|
|
34
|
+
`details` text,
|
|
35
|
+
`resolved_at` integer,
|
|
36
|
+
FOREIGN KEY (`monitor_id`) REFERENCES `monitors`(`id`) ON UPDATE no action ON DELETE cascade,
|
|
37
|
+
FOREIGN KEY (`suppressed_by_window_id`) REFERENCES `suppression_windows`(`id`) ON UPDATE no action ON DELETE set null,
|
|
38
|
+
FOREIGN KEY (`acked_by`) REFERENCES `people`(`id`) ON UPDATE no action ON DELETE set null,
|
|
39
|
+
FOREIGN KEY (`deferred_route_id`) REFERENCES `routes`(`id`) ON UPDATE no action ON DELETE set null
|
|
40
|
+
);
|
|
41
|
+
--> statement-breakpoint
|
|
42
|
+
INSERT INTO `__new_alerts` SELECT `id`, `key`, `active_key`, `monitor_id`, `state`, `severity`, `first_fired_at`, `last_seen_at`, `grace_until`, `suppressed_by_alert_id`, `suppressed_by_window_id`, `unsuppressed_at`, `awaiting_confirmation`, `acked_by`, `acked_at`, `silenced_until`, `escalation_step`, `next_escalation_at`, `deferred_until`, `deferred_route_id`, `message`, `details`, `resolved_at` FROM `alerts`;--> statement-breakpoint
|
|
43
|
+
DROP TABLE `alerts`;--> statement-breakpoint
|
|
44
|
+
ALTER TABLE `__new_alerts` RENAME TO `alerts`;--> statement-breakpoint
|
|
45
|
+
CREATE UNIQUE INDEX `alerts_live_key_idx` ON `alerts` (`active_key`);--> statement-breakpoint
|
|
46
|
+
CREATE INDEX `alerts_key_idx` ON `alerts` (`key`);
|
|
@@ -127,6 +127,13 @@
|
|
|
127
127
|
"when": 1783300000000,
|
|
128
128
|
"tag": "0017_alerting",
|
|
129
129
|
"breakpoints": true
|
|
130
|
+
},
|
|
131
|
+
{
|
|
132
|
+
"idx": 18,
|
|
133
|
+
"version": "6",
|
|
134
|
+
"when": 1783400000000,
|
|
135
|
+
"tag": "0018_drop_alert_policy_snapshot",
|
|
136
|
+
"breakpoints": true
|
|
130
137
|
}
|
|
131
138
|
]
|
|
132
|
-
}
|
|
139
|
+
}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@celilo/cli",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.15.0",
|
|
4
4
|
"description": "Celilo — home lab orchestration CLI",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"bin": {
|
|
@@ -57,10 +57,10 @@
|
|
|
57
57
|
},
|
|
58
58
|
"dependencies": {
|
|
59
59
|
"@aws-sdk/client-s3": "^3.1024.0",
|
|
60
|
-
"@celilo/capabilities": "^0.9.
|
|
60
|
+
"@celilo/capabilities": "^0.9.1",
|
|
61
61
|
"@celilo/cli-display": "^0.1.9",
|
|
62
|
-
"@celilo/core": "^0.
|
|
63
|
-
"@celilo/event-bus": "^0.1.
|
|
62
|
+
"@celilo/core": "^0.2.0",
|
|
63
|
+
"@celilo/event-bus": "^0.1.9",
|
|
64
64
|
"@clack/prompts": "^1.1.0",
|
|
65
65
|
"ajv": "^8.18.0",
|
|
66
66
|
"drizzle-orm": "^0.36.4",
|
|
@@ -516,7 +516,7 @@ describe('register_route — public DNS wiring (D1/M1 #328)', () => {
|
|
|
516
516
|
};
|
|
517
517
|
}
|
|
518
518
|
|
|
519
|
-
test('registers the public A record
|
|
519
|
+
test('registers the public A record for a NEW hostname without supplying an IP', async () => {
|
|
520
520
|
const { ops } = makeRouteOps();
|
|
521
521
|
const { registrar, calls } = makeRegistrar();
|
|
522
522
|
const cap = createPublicWeb({
|
|
@@ -530,7 +530,6 @@ describe('register_route — public DNS wiring (D1/M1 #328)', () => {
|
|
|
530
530
|
dnsManagedDomains: ['example.com'],
|
|
531
531
|
// biome-ignore lint/suspicious/noExplicitAny: minimal registrar stub
|
|
532
532
|
dnsRegistrar: registrar as any,
|
|
533
|
-
externalIp: '100.100.0.100',
|
|
534
533
|
});
|
|
535
534
|
|
|
536
535
|
await cap.register_route({
|
|
@@ -539,7 +538,9 @@ describe('register_route — public DNS wiring (D1/M1 #328)', () => {
|
|
|
539
538
|
hostname: 'nexus.example.com',
|
|
540
539
|
});
|
|
541
540
|
|
|
542
|
-
|
|
541
|
+
// No `ip` — the registrar publishes the source address of the update
|
|
542
|
+
// request, which is by construction the address the internet must dial.
|
|
543
|
+
expect(calls).toEqual([{ fqdn: 'nexus.example.com', ip: undefined }]);
|
|
543
544
|
});
|
|
544
545
|
|
|
545
546
|
test('skips the default hostname (caddy already registered it at install)', async () => {
|
|
@@ -556,7 +557,6 @@ describe('register_route — public DNS wiring (D1/M1 #328)', () => {
|
|
|
556
557
|
dnsManagedDomains: ['example.com'],
|
|
557
558
|
// biome-ignore lint/suspicious/noExplicitAny: minimal registrar stub
|
|
558
559
|
dnsRegistrar: registrar as any,
|
|
559
|
-
externalIp: '100.100.0.100',
|
|
560
560
|
});
|
|
561
561
|
|
|
562
562
|
await cap.register_route({ type: 'static', path: '/', hostname: 'www.example.com' });
|
|
@@ -568,7 +568,13 @@ describe('register_route — public DNS wiring (D1/M1 #328)', () => {
|
|
|
568
568
|
// hostname that can't be wired FAILS the deploy instead of silently
|
|
569
569
|
// reporting success (the "served but unreachable" anti-pattern).
|
|
570
570
|
|
|
571
|
-
|
|
571
|
+
// The regression guard for #464. This case used to THROW "caddy has no known
|
|
572
|
+
// external IP" — and it threw on every real fleet, because the stored copy it
|
|
573
|
+
// demanded was never populated: caddy produced a `public_ip` hook output that
|
|
574
|
+
// its manifest never declared, so the framework discarded it. A correct fleet
|
|
575
|
+
// with correct public DNS could not register a new hostname. Registration must
|
|
576
|
+
// not depend on knowing the external IP at all.
|
|
577
|
+
test('registers a NEW hostname even though no external IP is known anywhere', async () => {
|
|
572
578
|
const { ops } = makeRouteOps();
|
|
573
579
|
const { registrar, calls } = makeRegistrar();
|
|
574
580
|
const cap = createPublicWeb({
|
|
@@ -582,13 +588,11 @@ describe('register_route — public DNS wiring (D1/M1 #328)', () => {
|
|
|
582
588
|
dnsManagedDomains: ['example.com'],
|
|
583
589
|
// biome-ignore lint/suspicious/noExplicitAny: minimal registrar stub
|
|
584
590
|
dnsRegistrar: registrar as any,
|
|
585
|
-
// externalIp omitted
|
|
586
591
|
});
|
|
587
592
|
|
|
588
|
-
await
|
|
589
|
-
|
|
590
|
-
).
|
|
591
|
-
expect(calls).toHaveLength(0);
|
|
593
|
+
await cap.register_route({ type: 'static', path: '/', hostname: 'nexus.example.com' });
|
|
594
|
+
|
|
595
|
+
expect(calls).toEqual([{ fqdn: 'nexus.example.com', ip: undefined }]);
|
|
592
596
|
});
|
|
593
597
|
|
|
594
598
|
test('fails loudly when no dns_registrar is available for a NEW hostname', async () => {
|
|
@@ -602,8 +606,7 @@ describe('register_route — public DNS wiring (D1/M1 #328)', () => {
|
|
|
602
606
|
hostnames: ['www.example.com'],
|
|
603
607
|
caddyModuleId: 'caddy',
|
|
604
608
|
dnsManagedDomains: ['example.com'],
|
|
605
|
-
// dnsRegistrar
|
|
606
|
-
externalIp: '100.100.0.100',
|
|
609
|
+
// dnsRegistrar omitted
|
|
607
610
|
});
|
|
608
611
|
|
|
609
612
|
await expect(
|
|
@@ -629,7 +632,6 @@ describe('register_route — public DNS wiring (D1/M1 #328)', () => {
|
|
|
629
632
|
dnsManagedDomains: ['example.com'],
|
|
630
633
|
// biome-ignore lint/suspicious/noExplicitAny: minimal registrar stub
|
|
631
634
|
dnsRegistrar: registrar as any,
|
|
632
|
-
externalIp: '100.100.0.100',
|
|
633
635
|
});
|
|
634
636
|
|
|
635
637
|
await expect(
|
|
@@ -656,7 +658,6 @@ describe('register_route — public DNS wiring (D1/M1 #328)', () => {
|
|
|
656
658
|
dnsManagedDomains: ['example.com'],
|
|
657
659
|
// biome-ignore lint/suspicious/noExplicitAny: minimal registrar stub
|
|
658
660
|
dnsRegistrar: registrar as any,
|
|
659
|
-
externalIp: '100.100.0.100',
|
|
660
661
|
// biome-ignore lint/suspicious/noExplicitAny: minimal dns_internal stub
|
|
661
662
|
dnsInternal: dnsInternal as any,
|
|
662
663
|
});
|
|
@@ -687,7 +688,6 @@ describe('register_route — public DNS wiring (D1/M1 #328)', () => {
|
|
|
687
688
|
dnsManagedDomains: ['example.com'],
|
|
688
689
|
// biome-ignore lint/suspicious/noExplicitAny: minimal registrar stub
|
|
689
690
|
dnsRegistrar: registrar as any,
|
|
690
|
-
externalIp: '100.100.0.100',
|
|
691
691
|
firewallNatIp: '192.168.0.253',
|
|
692
692
|
// biome-ignore lint/suspicious/noExplicitAny: minimal dns_internal stub
|
|
693
693
|
dnsInternal: dnsInternal as any,
|
|
@@ -8,6 +8,8 @@
|
|
|
8
8
|
|
|
9
9
|
import { getDb } from '../../db/client';
|
|
10
10
|
import { renderAlertTable, toAlertRow } from '../../services/alerting/format';
|
|
11
|
+
import { listMonitors } from '../../services/alerting/monitors';
|
|
12
|
+
import { policyForAlert } from '../../services/alerting/notify-deps';
|
|
11
13
|
import { loadAllLiveAlerts } from '../../services/alerting/store';
|
|
12
14
|
import type { CommandResult } from '../types';
|
|
13
15
|
|
|
@@ -20,6 +22,7 @@ export async function handleAlertsList(
|
|
|
20
22
|
const live = loadAllLiveAlerts(db);
|
|
21
23
|
|
|
22
24
|
if (flags.json) {
|
|
25
|
+
const monitorsById = new Map(listMonitors(db).map((m) => [m.id, m.target]));
|
|
23
26
|
return {
|
|
24
27
|
success: true,
|
|
25
28
|
message: JSON.stringify(
|
|
@@ -27,6 +30,13 @@ export async function handleAlertsList(
|
|
|
27
30
|
key: alert.key,
|
|
28
31
|
state: alert.state,
|
|
29
32
|
severity: alert.severity,
|
|
33
|
+
// Who owns this alert and who it would page. Without these an
|
|
34
|
+
// operator told "1 live alert has no escalation policy" had no way to
|
|
35
|
+
// find out which one, from any command celilo offered (#481).
|
|
36
|
+
monitor: monitorsById.get(alert.monitorId) ?? null,
|
|
37
|
+
escalationPolicy: policyForAlert(db, alert)?.name ?? null,
|
|
38
|
+
escalationStep: alert.escalationStep,
|
|
39
|
+
nextEscalationAt: alert.nextEscalationAt,
|
|
30
40
|
firstFiredAt: alert.firstFiredAt,
|
|
31
41
|
lastSeenAt: alert.lastSeenAt,
|
|
32
42
|
suppressed: alert.state === 'suppressed',
|
|
@@ -104,16 +104,12 @@ export async function handleAlertsPoll(
|
|
|
104
104
|
|
|
105
105
|
responder.stop();
|
|
106
106
|
|
|
107
|
-
// Rejections are worth surfacing rather than counting silently: a run of
|
|
108
|
-
// them means someone is replying and not being heard, which looks identical
|
|
109
|
-
// to the alerting system being broken.
|
|
110
107
|
const parts = [
|
|
111
108
|
`${report.transportsPolled} transport(s)`,
|
|
112
109
|
`${report.messagesRead} message(s)`,
|
|
113
110
|
`${report.acked} acked`,
|
|
114
111
|
];
|
|
115
|
-
if (report.
|
|
116
|
-
if (report.ignored > 0) parts.push(`${report.ignored} from unknown senders`);
|
|
112
|
+
if (report.unheard.length > 0) parts.push(`${report.unheard.length} NOT HEARD`);
|
|
117
113
|
if (report.broadcast > 0) parts.push(`${report.broadcast} told someone has it`);
|
|
118
114
|
if (report.answered > 0) parts.push(`${report.answered} deploy question(s) answered`);
|
|
119
115
|
if (ask.asked > 0) parts.push(`${ask.asked} question(s) asked`);
|
|
@@ -121,9 +117,19 @@ export async function handleAlertsPoll(
|
|
|
121
117
|
// indistinguishable from a deploy that is merely slow.
|
|
122
118
|
if (ask.failed > 0) parts.push(`${ask.failed} question(s) COULD NOT BE DELIVERED`);
|
|
123
119
|
|
|
120
|
+
// The summary line counts; these lines say WHY. Without them "0 message(s)"
|
|
121
|
+
// means both "nobody replied" and "the reply was read and thrown away", and
|
|
122
|
+
// an operator has no way to tell which — the ambiguity that made the Signal
|
|
123
|
+
// ack path take a week to diagnose.
|
|
124
|
+
const detail = [
|
|
125
|
+
...report.failures.map((f) => `${f.transportModuleId} COULD NOT BE READ: ${f.error}`),
|
|
126
|
+
...report.unheard.map((u) => `not heard from ${u.senderAddress}: ${u.reason}`),
|
|
127
|
+
...askErrors,
|
|
128
|
+
];
|
|
129
|
+
|
|
124
130
|
const message = `inbound poll: ${parts.join(', ')}`;
|
|
125
131
|
return {
|
|
126
132
|
success: true,
|
|
127
|
-
message:
|
|
133
|
+
message: detail.length > 0 ? `${message}\n ${detail.join('\n ')}` : message,
|
|
128
134
|
};
|
|
129
135
|
}
|
|
@@ -10,19 +10,15 @@
|
|
|
10
10
|
* report counts.
|
|
11
11
|
*/
|
|
12
12
|
|
|
13
|
-
import { eq } from 'drizzle-orm';
|
|
14
13
|
import { getDb } from '../../db/client';
|
|
15
|
-
import {
|
|
14
|
+
import { modules } from '../../db/schema';
|
|
16
15
|
import { runBuiltinCheckForMonitor } from '../../services/alerting/builtin-source';
|
|
17
16
|
import { loadModuleCoverage } from '../../services/alerting/coverage-source';
|
|
18
17
|
import { modulesInDeployWindow } from '../../services/alerting/deploy-hooks';
|
|
19
18
|
import { listMonitors } from '../../services/alerting/monitors';
|
|
20
|
-
import
|
|
21
|
-
import { listPeople, listPolicySteps, listRoutes } from '../../services/alerting/people';
|
|
19
|
+
import { buildNotifyDeps } from '../../services/alerting/notify-deps';
|
|
22
20
|
import type { SuppressionTopology } from '../../services/alerting/suppression';
|
|
23
21
|
import { runSweep } from '../../services/alerting/sweep-runner';
|
|
24
|
-
import { mintDelivery } from '../../services/alerting/tokens';
|
|
25
|
-
import { loadNotificationTransport } from '../../services/alerting/transport-loader';
|
|
26
22
|
import { getModuleSystems } from '../../services/deployed-systems';
|
|
27
23
|
import { runModuleHealthCheck } from '../../services/health-runner';
|
|
28
24
|
import type { CommandResult } from '../types';
|
|
@@ -30,13 +26,6 @@ import type { CommandResult } from '../types';
|
|
|
30
26
|
/** Grace window before a newly-fired alert may notify. */
|
|
31
27
|
const DEFAULT_GRACE_MS = 60_000;
|
|
32
28
|
|
|
33
|
-
/**
|
|
34
|
-
* How long a reply token stays valid. A day is long enough that someone who
|
|
35
|
-
* sees a page overnight can still act on it in the morning, and short enough
|
|
36
|
-
* that a token found later is useless.
|
|
37
|
-
*/
|
|
38
|
-
const TOKEN_TTL_MS = 24 * 60 * 60_000;
|
|
39
|
-
|
|
40
29
|
/**
|
|
41
30
|
* Deployment topology for suppression, read once per sweep.
|
|
42
31
|
*
|
|
@@ -59,69 +48,6 @@ function loadTopology(db: ReturnType<typeof getDb>): SuppressionTopology {
|
|
|
59
48
|
return { moduleSystems, zoneProviders: [] };
|
|
60
49
|
}
|
|
61
50
|
|
|
62
|
-
/**
|
|
63
|
-
* Assemble everything needed to page for one alert, or null when nobody can be.
|
|
64
|
-
*
|
|
65
|
-
* Returning null rather than an empty policy is deliberate: "no escalation
|
|
66
|
-
* policy assigned" and "policy exists but nobody is eligible" are different
|
|
67
|
-
* situations, and only the second is worth an escalation decision.
|
|
68
|
-
*/
|
|
69
|
-
function buildNotifyDeps(db: ReturnType<typeof getDb>, alert: Alert): NotifyDeps | null {
|
|
70
|
-
if (!alert.escalationPolicyId) return null;
|
|
71
|
-
|
|
72
|
-
const policy = db
|
|
73
|
-
.select()
|
|
74
|
-
.from(escalationPolicies)
|
|
75
|
-
.where(eq(escalationPolicies.id, alert.escalationPolicyId))
|
|
76
|
-
.get();
|
|
77
|
-
if (!policy) return null;
|
|
78
|
-
|
|
79
|
-
const steps = listPolicySteps(db, policy.id).map((step) => ({
|
|
80
|
-
stepIndex: step.stepIndex,
|
|
81
|
-
routeId: step.routeId,
|
|
82
|
-
delayMinutes: step.delayMinutes,
|
|
83
|
-
}));
|
|
84
|
-
if (steps.length === 0) return null;
|
|
85
|
-
|
|
86
|
-
const routeRows = listRoutes(db);
|
|
87
|
-
const people = listPeople(db);
|
|
88
|
-
|
|
89
|
-
return {
|
|
90
|
-
steps,
|
|
91
|
-
routes: new Map(
|
|
92
|
-
routeRows.map((r) => [
|
|
93
|
-
r.id,
|
|
94
|
-
{ id: r.id, severityFloor: r.severityFloor, enabled: r.enabled },
|
|
95
|
-
]),
|
|
96
|
-
),
|
|
97
|
-
routeDetails: new Map(routeRows.map((r) => [r.id, r])),
|
|
98
|
-
quietHoursByPerson: new Map(
|
|
99
|
-
people
|
|
100
|
-
.filter((p) => p.quietHoursStart && p.quietHoursEnd)
|
|
101
|
-
.map((p) => [
|
|
102
|
-
p.id,
|
|
103
|
-
{
|
|
104
|
-
personId: p.id,
|
|
105
|
-
start: p.quietHoursStart,
|
|
106
|
-
end: p.quietHoursEnd,
|
|
107
|
-
timezone: p.timezone,
|
|
108
|
-
},
|
|
109
|
-
]),
|
|
110
|
-
),
|
|
111
|
-
bypassQuietHours: policy.bypassQuietHours,
|
|
112
|
-
transportFor: (route) => loadNotificationTransport(db, route.transportModuleId),
|
|
113
|
-
mintToken: (alertId, routeId) =>
|
|
114
|
-
mintDelivery(db, {
|
|
115
|
-
kind: 'alert',
|
|
116
|
-
targetId: alertId,
|
|
117
|
-
routeId,
|
|
118
|
-
now: new Date(),
|
|
119
|
-
ttlMs: TOKEN_TTL_MS,
|
|
120
|
-
}).token,
|
|
121
|
-
now: new Date(),
|
|
122
|
-
};
|
|
123
|
-
}
|
|
124
|
-
|
|
125
51
|
export async function handleAlertsSweep(): Promise<CommandResult> {
|
|
126
52
|
const db = getDb();
|
|
127
53
|
const monitorRows = listMonitors(db);
|
|
@@ -132,7 +58,7 @@ export async function handleAlertsSweep(): Promise<CommandResult> {
|
|
|
132
58
|
monitorDeps: {
|
|
133
59
|
runModuleCheck: (moduleId: string) =>
|
|
134
60
|
runModuleHealthCheck(moduleId, db, { unattended: true, noInteractive: true }),
|
|
135
|
-
runBuiltinCheck: (category) => runBuiltinCheckForMonitor(category),
|
|
61
|
+
runBuiltinCheck: (category) => runBuiltinCheckForMonitor(category, db),
|
|
136
62
|
loadModuleCoverage: () => loadModuleCoverage(db),
|
|
137
63
|
now: () => new Date(),
|
|
138
64
|
graceMs: DEFAULT_GRACE_MS,
|
|
@@ -140,7 +66,7 @@ export async function handleAlertsSweep(): Promise<CommandResult> {
|
|
|
140
66
|
loadTopology: () => loadTopology(db),
|
|
141
67
|
loadDeployWindowModules: () => modulesInDeployWindow(db),
|
|
142
68
|
isSuppressible: (alert) => suppressibleByMonitor.get(alert.monitorId) ?? true,
|
|
143
|
-
notifyDepsFor: (alert) => buildNotifyDeps(db, alert),
|
|
69
|
+
notifyDepsFor: (alert) => buildNotifyDeps(db, alert, new Date()),
|
|
144
70
|
now: () => new Date(),
|
|
145
71
|
});
|
|
146
72
|
|
|
@@ -160,7 +86,7 @@ export async function handleAlertsSweep(): Promise<CommandResult> {
|
|
|
160
86
|
if (report.deferred > 0) parts.push(`${report.deferred} deferred`);
|
|
161
87
|
if (report.deferredDelivered > 0) parts.push(`${report.deferredDelivered} deferred-delivered`);
|
|
162
88
|
if (report.failed > 0) parts.push(`${report.failed} FAILED`);
|
|
163
|
-
if (report.noPolicy > 0) parts.push(`${report.noPolicy} no-policy`);
|
|
89
|
+
if (report.noPolicy.length > 0) parts.push(`${report.noPolicy.length} no-policy`);
|
|
164
90
|
|
|
165
91
|
const lines = [`alert sweep: ${parts.join(', ')}`];
|
|
166
92
|
|
|
@@ -176,10 +102,25 @@ export async function handleAlertsSweep(): Promise<CommandResult> {
|
|
|
176
102
|
for (const failure of report.failures) {
|
|
177
103
|
lines.push(` FAILED ${failure}`);
|
|
178
104
|
}
|
|
179
|
-
|
|
105
|
+
// Name the alert AND the monitor that owns it. The previous wording printed a
|
|
106
|
+
// count and a template command, which is unusable: the operator cannot tell
|
|
107
|
+
// which of their live alerts is the unrouted one, so the only way to act on it
|
|
108
|
+
// is to assign a policy to every monitor and hope (#481).
|
|
109
|
+
//
|
|
110
|
+
// The wording says "no policy, or a policy with no steps" because that is
|
|
111
|
+
// genuinely all the sweep knows here — both produce a null NotifyDeps. Naming
|
|
112
|
+
// only the first would send an operator whose policy is merely empty chasing
|
|
113
|
+
// an assignment they have already made, which is the failure being fixed.
|
|
114
|
+
if (report.noPolicy.length > 0) {
|
|
180
115
|
lines.push(
|
|
181
|
-
` ${report.noPolicy} live alert(s)
|
|
116
|
+
` ${report.noPolicy.length} live alert(s) reach nobody — no escalation policy, or a policy with no steps:`,
|
|
182
117
|
);
|
|
118
|
+
for (const { alertKey, monitor } of report.noPolicy) {
|
|
119
|
+
lines.push(` ${alertKey} (monitor: ${monitor})`);
|
|
120
|
+
lines.push(` celilo escalation-policy assign <policy> ${monitor}`);
|
|
121
|
+
}
|
|
122
|
+
lines.push(' Which policy each monitor uses: celilo monitor list');
|
|
123
|
+
lines.push(' Whether that policy has steps: celilo escalation-policy list');
|
|
183
124
|
}
|
|
184
125
|
|
|
185
126
|
return { success: true, message: lines.join('\n') };
|
|
@@ -0,0 +1,65 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* `celilo backup sweep` — one pass of the scheduled backup runner.
|
|
3
|
+
*
|
|
4
|
+
* Invoked by the event-bus dispatcher on `timer.tick.1h`, not by a human. It is
|
|
5
|
+
* a normal command so the dispatcher's existing subprocess isolation, retry,
|
|
6
|
+
* and timeout apply unchanged.
|
|
7
|
+
*
|
|
8
|
+
* Thin adapter (Rule 10.5): compose the real dependencies, call runBackupSweep,
|
|
9
|
+
* report counts.
|
|
10
|
+
*/
|
|
11
|
+
|
|
12
|
+
import {
|
|
13
|
+
createModuleBackup,
|
|
14
|
+
findBackupEligibleModules,
|
|
15
|
+
isBackupDue,
|
|
16
|
+
} from '../../services/backup-create';
|
|
17
|
+
import { pruneBackupsForModule } from '../../services/backup-retention';
|
|
18
|
+
import { type BackupSweepReport, runBackupSweep } from '../../services/backup-sweep';
|
|
19
|
+
import type { CommandResult } from '../types';
|
|
20
|
+
|
|
21
|
+
export async function handleBackupSweep(): Promise<CommandResult> {
|
|
22
|
+
const report = await runBackupSweep({
|
|
23
|
+
listEligible: () =>
|
|
24
|
+
findBackupEligibleModules().map(({ module, manifest }) => ({ id: module.id, manifest })),
|
|
25
|
+
isDue: (moduleId, schedule) => isBackupDue(moduleId, schedule),
|
|
26
|
+
backup: (moduleId) => createModuleBackup(moduleId),
|
|
27
|
+
prune: async ({ id, manifest }) => {
|
|
28
|
+
const retention = manifest.backup?.retention;
|
|
29
|
+
if (!retention) return;
|
|
30
|
+
await pruneBackupsForModule(id, {
|
|
31
|
+
count: retention.count,
|
|
32
|
+
maxAgeDays: retention.max_age_days,
|
|
33
|
+
});
|
|
34
|
+
},
|
|
35
|
+
});
|
|
36
|
+
|
|
37
|
+
const summary = formatReport(report);
|
|
38
|
+
if (report.failures.length > 0) {
|
|
39
|
+
return { success: false, error: summary };
|
|
40
|
+
}
|
|
41
|
+
return { success: true, message: summary };
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
/**
|
|
45
|
+
* A quiet sweep stays quiet: counts that are zero are omitted. But a failure or
|
|
46
|
+
* a lock-skip is always named, never aggregated away — "0 backed up" and
|
|
47
|
+
* nothing else is indistinguishable from "nothing was due", which is exactly
|
|
48
|
+
* how a backup runner that has silently stopped working looks.
|
|
49
|
+
*/
|
|
50
|
+
function formatReport(report: BackupSweepReport): string {
|
|
51
|
+
const parts = [`${report.backedUp.length} backed up`];
|
|
52
|
+
if (report.skippedNotDue.length > 0) parts.push(`${report.skippedNotDue.length} not due`);
|
|
53
|
+
if (report.skippedManual.length > 0) parts.push(`${report.skippedManual.length} manual`);
|
|
54
|
+
if (report.skippedLocked.length > 0) parts.push(`${report.skippedLocked.length} locked`);
|
|
55
|
+
if (report.failures.length > 0) parts.push(`${report.failures.length} FAILED`);
|
|
56
|
+
|
|
57
|
+
const lines = [`backup sweep: ${parts.join(', ')}`];
|
|
58
|
+
for (const moduleId of report.skippedLocked) {
|
|
59
|
+
lines.push(` skipped ${moduleId}: another operation is in flight — retrying next tick`);
|
|
60
|
+
}
|
|
61
|
+
for (const failure of report.failures) {
|
|
62
|
+
lines.push(` FAILED ${failure.moduleId}: ${failure.error}`);
|
|
63
|
+
}
|
|
64
|
+
return lines.join('\n');
|
|
65
|
+
}
|