@celilo/cli 1.14.0 → 2.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (57) hide show
  1. package/CELILO_CORE_MODULES.md +1 -1
  2. package/CELILO_SUBSYSTEMS.md +25 -3
  3. package/README.md +0 -2
  4. package/drizzle/0030_drop_module_builds_environment.sql +8 -0
  5. package/drizzle/meta/_journal.json +8 -1
  6. package/package.json +3 -3
  7. package/src/capabilities/public-web-publish.test.ts +18 -0
  8. package/src/cli/commands/alerts-sweep.ts +3 -0
  9. package/src/cli/commands/monitor.ts +15 -2
  10. package/src/cli/commands/system-doctor.test.ts +121 -1
  11. package/src/cli/commands/system-doctor.ts +151 -1
  12. package/src/cli/completion.ts +9 -2
  13. package/src/cli/index.ts +1 -1
  14. package/src/console/control-plane-boundary.test.ts +82 -4
  15. package/src/db/schema.ts +0 -1
  16. package/src/hooks/capability-loader.ts +14 -0
  17. package/src/hooks/executor.ts +110 -17
  18. package/src/hooks/hook-jail-toolchain-reach.test.ts +224 -0
  19. package/src/hooks/hook-jail-unreachability.test.ts +28 -2
  20. package/src/hooks/hook-protocol.ts +44 -0
  21. package/src/hooks/hook-runner-entry.ts +23 -0
  22. package/src/hooks/hook-runner.ts +10 -0
  23. package/src/hooks/hook-trespass.test.ts +9 -3
  24. package/src/hooks/jail-browser-launch-flags.test.ts +34 -0
  25. package/src/hooks/jail.test.ts +92 -0
  26. package/src/hooks/jail.ts +128 -11
  27. package/src/hooks/mount-set.test.ts +28 -6
  28. package/src/hooks/mount-set.ts +34 -20
  29. package/src/hooks/remote-broker.test.ts +350 -0
  30. package/src/hooks/remote-broker.ts +404 -0
  31. package/src/hooks/run-named-hook.ts +2 -0
  32. package/src/hooks/test-fixtures/jail-probe-hook.ts +14 -1
  33. package/src/hooks/test-fixtures/jail-toolchain-hook.ts +227 -0
  34. package/src/hooks/test-fixtures/remote-bridge-probe.ts +82 -0
  35. package/src/hooks/unjailed-lint.test.ts +251 -0
  36. package/src/hooks/unjailed-lint.ts +395 -0
  37. package/src/policy/module-business-baseline.ts +13 -1
  38. package/src/policy/module-script-scan.ts +60 -1
  39. package/src/policy/no-hand-built-ssh.test.ts +39 -1
  40. package/src/policy/no-module-business-in-core.test.ts +1 -1
  41. package/src/services/alerting/hook-jail.test.ts +66 -0
  42. package/src/services/alerting/hook-jail.ts +70 -0
  43. package/src/services/alerting/run-monitor.test.ts +62 -0
  44. package/src/services/alerting/run-monitor.ts +12 -0
  45. package/src/services/alerting/sweep-runner.test.ts +1 -0
  46. package/src/services/backup-create.ts +3 -0
  47. package/src/services/backup-restore.ts +2 -0
  48. package/src/services/deploy-ansible.ts +9 -1
  49. package/src/services/health-runner.ts +2 -0
  50. package/src/services/module-build.test.ts +1 -64
  51. package/src/services/module-build.ts +10 -86
  52. package/src/services/module-deploy.ts +20 -0
  53. package/src/services/remote-access.test.ts +139 -0
  54. package/src/services/remote-access.ts +98 -0
  55. package/src/services/restore-from-file.ts +6 -1
  56. package/src/services/static-content-converge.test.ts +338 -0
  57. package/src/services/static-content-converge.ts +299 -0
@@ -59,7 +59,7 @@ Each entry: `module id` — what it is — **provides** / **requires** capabilit
59
59
  - **celilo-apt-repo** — Debian apt repository (reprepro + Bun HTTP server) serving the celilo `.deb` at apt.celilo.computer. **provides:** `apt_publish`. **requires:** `public_web`, `dns_registrar`.
60
60
  - **signal** — bidirectional Signal transport for alerts and deploy-interview questions; runs signal-cli in daemon mode with its JSON-RPC socket bound to the host's own address (never public) and **`--receive-mode=manual`**, which is load-bearing: signal-cli's default (`on-start`) leaves the daemon permanently receiving, so it drains every reply into an SSE stream nothing is attached to and REFUSES celilo's `receive` call — replies arrive and are unreadable, while `send` and every health check keep passing. Enrolled as a SECONDARY DEVICE of an existing Signal account rather than registering its own number — Signal blocks most VOIP ranges and bans bot-ish registrations. Recipient addresses live on celilo routes, not in module config, so adding a person never requires a redeploy. Runs on x86_64 and aarch64. `libsignal-client` ships no linux-aarch64 native, so celilo builds one (`modules/signal/build/`) and installs it as a `libsignal-jni` .deb on ARM hosts; x86_64 uses the JAR's bundled native. **provides:** `notification` (`send`, `receive`). **requires:** no capabilities — a transport that depended on the proxy, registrar or firewall could not tell you those were broken — and a system in the **`secure-mgmt`** zone: it holds a linked Signal account (the operator's own messaging identity and keys), and its job is to observe every tier while depending on none, which is what the control-plane zone is for. See `openspec/changes/add-alerting/`.
61
61
  - **celilo-website** — public docs site (static Astro) served via Caddy on celilo.computer. **requires:** `public_web`, `dns_registrar`.
62
- - **celilo-web-console** — the read-mostly operator console: the fleet drawn by zone, a live module roster, and the alert and backup state that says what needs attention. **Not yet deployed** — the manifest, the SPA, the console server, the read verbs and the `control_plane_api` capability exist; the `on_install` that enrols the principal, and the e2e suite, do not. It holds **no database handle**: every fact it displays arrives over the SSH remote API as a principal whose grants derive from the `COMMANDS` registry's read-only classifier, because an in-process console would hold the whole database including the encrypted secret store and be constrained only by its own code. **⚠️ `secure-mgmt`, and the placement is a security boundary rather than a preference:** celilo reaches into every data-plane zone by trust and no data-plane zone reaches back, so the ONLY client path in is the control-plane VPN. It registers no web route, requests no port forward, takes no `natIp` or ingress address, and gets no DNS record resolvable from a data-plane zone. `apps/celilo/src/console/control-plane-boundary.test.ts` asserts that on every pull request rather than only in e2e, because the way it erodes is a one-line manifest edit adding `private_web` so someone can reach it without bringing up the VPN. It offers no deploy, uninstall, pause, restore or config control: those can raise an interview question, a browser has no responder, and they are ABSENT rather than disabled. **provides:** nothing. **requires:** `idp`, `control_plane_api`. See `openspec/changes/web-ui-console/`.
62
+ - **celilo-web-console** — the read-mostly operator console: the fleet drawn by zone, a live module roster, and the alert and backup state that says what needs attention. **Deployable, not yet servable.** The manifest, the SPA, the console server, the read verbs, the `control_plane_api` capability, the `on_install` that enrols the principal, and a full deployable body (`terraform/`, `ansible/` with a role, config variables, and a `build:` step compiling a per-architecture bun binary plus the SPA) all exist. What does not exist is an OIDC client: the console refuses to serve without an identity provider, and nothing creates one yet, so a deployed unit starts, fails its start limit and sits in `failed` with the reason in its journal. That is deliberate rather than hidden — leaving the service disabled would make an unfinished module look like a clean converge. The e2e suite does not exist either. It holds **no database handle**: every fact it displays arrives over the SSH remote API as a principal whose grants derive from the `COMMANDS` registry's read-only classifier, because an in-process console would hold the whole database including the encrypted secret store and be constrained only by its own code. **⚠️ `secure-mgmt`, and the placement is a security boundary rather than a preference:** celilo reaches into every data-plane zone by trust and no data-plane zone reaches back, so the ONLY client path in is the control-plane VPN. It registers no web route, requests no port forward, takes no `natIp` or ingress address, and gets no DNS record resolvable from a data-plane zone. `apps/celilo/src/console/control-plane-boundary.test.ts` asserts that on every pull request rather than only in e2e, because the way it erodes is a one-line manifest edit adding `private_web` so someone can reach it without bringing up the VPN. It offers no deploy, uninstall, pause, restore or config control: those can raise an interview question, a browser has no responder, and they are ABSENT rather than disabled. **provides:** nothing. **requires:** `idp`, `control_plane_api`. See `openspec/changes/web-ui-console/`.
63
63
  - **celilo-canary** — a deliberately minimal nginx serving one static page in **`dmz`**, permanently deployed, whose health check IS the assertion that the deploy path still works. Nothing about it is interesting except that it is *always there*: modules already deployed keep running when the pipeline breaks, so without a canary a regression in IPAM allocation, Terraform provisioning, Ansible convergence or capability wiring stays invisible until the next real deploy — which is exactly when nobody wants to discover it. One deploy exercises all four, plus a live cross-module capability call. **Fleet-only, and the absence is the design**: it registers one route through `private_web` and requires nothing that reaches the perimeter, so there is no public record, no ACME certificate and no port forward. Its `health_check` probes nginx over systemd and its `/healthz` endpoint with **`probeHttp`, from the management server** rather than by SSHing in and running `curl` — the production `ubuntu-22.04-standard` LXC ships no curl, so the old form could not tell a missing binary from a dead service, and reaching the canary at its own address additionally catches a service bound only to loopback. **requires:** `private_web`.
64
64
 
65
65
  ## Git forge & CI pipeline
@@ -93,6 +93,26 @@ content module targets either with
93
93
  `capabilities.external_web ?? capabilities.public_web`. Both are
94
94
  **authoritative**: a publish that doesn't reach clients throws (#328).
95
95
 
96
+ - **Static-content converge (`/srv/www`, design D10)** —
97
+ `apps/celilo/src/services/static-content-converge.ts` (pure
98
+ `planStaticContent`/`buildStaticContentPlan` + `writeStaticContentVars` + the
99
+ `executeAnsible` caller `convergeStaticContent`). The declared `web_routes`
100
+ release set (slug, content hash, hostnames, the source dir core derives from
101
+ `modules.source_path`) is the desired state; the provider's Ansible converge
102
+ makes `/srv/www` match: content-hashed release dirs `/srv/www/<slug>-<hash>`,
103
+ an atomic symlink swap of `/srv/www/<slug>`, pruning beyond
104
+ `static_release_retention` (caddy manifest variable, operator config with
105
+ default 5 — the role never carries the number). Two callers, one plan: the
106
+ `public_web` capability's `convergeStaticContent` callback (tag-scoped run
107
+ through `executeAnsible`, so a publish returns only once the host matches)
108
+ and the provider's own deploy (vars written pre-Ansible in
109
+ `module-deploy.ts`, so a rebuilt host recovers with no consumer). The
110
+ content hash lands on EVERY route row sharing the slug. Replaced the
111
+ hand-built ssh tar pipe in `upload_static_assets` (celilo#1014); the
112
+ no-hand-built-SSH gate now scans `@celilo/capabilities` everywhere except
113
+ its primitive file (`module-script-scan.ts`, `scanCapabilityPackageSource`).
114
+ e2e: `modules/caddy/e2e/static-content-converge.test.ts`.
115
+
96
116
  ## Remote-ops primitives (the SSH seam — modules never hand-build SSH)
97
117
 
98
118
  Module hooks reach a remote box ONLY through these typed primitives
@@ -180,8 +200,10 @@ runner seam (`execRunner` real / `createMockRunner` for tests) lives in
180
200
  ## Hooks & deploy
181
201
 
182
202
  - **Hook executor / ABI** — `apps/celilo/src/hooks/executor.ts` (`invokeHook`, `executeHookScript`, `checkRequiredCapabilities`), types in `apps/celilo/src/hooks/types.ts` (`HookContext`, `HookDefinition`, `HookName`). Named-hook runner: `apps/celilo/src/hooks/run-named-hook.ts`. Manifest hook config: `apps/celilo/src/hooks/load-hook-config.ts`.
183
- - **Hook process boundary (a hook is a program celilo RUNS)** — `apps/celilo/src/hooks/hook-protocol.ts` (the NDJSON frame union, `HOOK_PROTOCOL_VERSION`, `serializeError`/`deserializeError`), `apps/celilo/src/hooks/broker.ts` (`startBroker`, `capabilityShape`), `apps/celilo/src/hooks/hook-runner.ts` (the spawned shim — the ONLY thing that `import()`s module code). `executeHookScript` spawns `bun hook-runner.ts` over a Unix socket instead of importing; the nine `invokeHook` call sites and `defineHook` are unchanged. **The broker does not know what a capability is**: it sends a shape descriptor built by the same own-string-key walk `wrapWithLogging` does (functions → `methods`, everything else → `data`, which is where `stampProvider`'s `providerModuleId` lives), and the shim rebuilds forwarding proxies from it — so an optional method a provider did not implement is absent rather than present-and-throwing, and `if (cap.registerTrustedSource)` keeps answering correctly. A socket rather than stdout because module scripts spawn subprocesses and a grandchild writing to fd 1 would corrupt the frame stream. Two consequences worth knowing: the child's environment is an **allow-list** (`hookChildEnv` — `PATH`, `HOME`, `LANG`, `TZ`, `TMPDIR`, `CELILO_HOOK_*`, `CELILO_DEBUG`), so a hook reading any other operator variable now gets `undefined`; and a timeout is a real SIGTERM-then-SIGKILL with the broker refusing further capability calls, replacing a `Promise.race` that cancelled nothing and let a "timed out" hook go on writing DNS and firewall state (celilo#1003). Capability PROVIDER factories still load in-process — they ARE the broker's implementation. Stages 1 and 2 of `openspec/changes/hook-process-boundary`; the filesystem is claimed by the jail below, the network is not claimed at all (stage 3, D12).
184
- - **Hook jail (a hook sees the paths it was given, and nothing else)** — `apps/celilo/src/hooks/mount-set.ts` (`deriveMountSet`, `toBwrapArgs`, `forbiddenPaths` — PURE, computes a filesystem view and touches nothing) and `apps/celilo/src/hooks/jail.ts` (`detectJailBackend`, `planJailedSpawn`, `realpathRequest`, `runtimeModulePathsFor`, `recordJailMode` — the half that touches the machine). `executeHookScript` spawns the shim under `bwrap` with the module's tree read-only, `state/` + `generated/` + this run's `screenshots/<run>` read-write on top of it, each contract-declared path input at its declared access, the broker's socket directory, the runtime and the `node_modules` the shim resolves through, `~/.ssh` read-only (stage 2 only — stage 3 drops it, D12), and `/tmp` a fresh tmpfs FIRST so it cannot erase the socket or a staged input. **The acceptance criterion is absence, not a check**: the module store and the data directory are simply not bound, so `master.key` and a sibling module give `ENOENT`. The set is DERIVED — a module cannot ask for more — and `bwrap` itself is never in it, because the AppArmor profile grants `userns` to `/usr/bin/bwrap` for anyone on the box (design D9). Backend detection RUNS bubblewrap rather than looking for it (four different denials all leave the binary in place), and the resulting mode is written to `hook-jail-mode.json` beside celilo's other per-machine state rather than logged, so a host that stops jailing is readable. `CELILO_HOOK_JAIL` = `auto` (default) / `required` / `off`. No backend on macOS yet (task 4.8), and the hook then runs unjailed with the mode recorded.
203
+ - **Hook process boundary (a hook is a program celilo RUNS)** — `apps/celilo/src/hooks/hook-protocol.ts` (the NDJSON frame union, `HOOK_PROTOCOL_VERSION`, `serializeError`/`deserializeError`), `apps/celilo/src/hooks/broker.ts` (`startBroker`, `capabilityShape`), `apps/celilo/src/hooks/hook-runner-entry.ts` + `apps/celilo/src/hooks/hook-runner.ts` (the spawned shim — the ONLY thing that `import()`s module code; the entry exists to install the advisory lint before the shim's import graph can ESM-load `node:fs`, see `unjailed-lint.ts`). `executeHookScript` spawns `bun hook-runner-entry.ts` over a Unix socket instead of importing; the nine `invokeHook` call sites and `defineHook` are unchanged. **The broker does not know what a capability is**: it sends a shape descriptor built by the same own-string-key walk `wrapWithLogging` does (functions → `methods`, everything else → `data`, which is where `stampProvider`'s `providerModuleId` lives), and the shim rebuilds forwarding proxies from it — so an optional method a provider did not implement is absent rather than present-and-throwing, and `if (cap.registerTrustedSource)` keeps answering correctly. A socket rather than stdout because module scripts spawn subprocesses and a grandchild writing to fd 1 would corrupt the frame stream. Two consequences worth knowing: the child's environment is an **allow-list** (`hookChildEnv` — how to run: `PATH`, `HOME`, `LANG`, `TZ`, `TMPDIR`; whom to trust: `NODE_EXTRA_CA_CERTS`, `SSL_CERT_FILE`, `SSL_CERT_DIR`; the proxy variables; the `CELILO_HOOK_*` channels; `CELILO_DEBUG` — `FORWARDED_ENV` in executor.ts is the source of truth), so a hook reading any other operator variable now gets `undefined`; and a timeout is a real SIGTERM-then-SIGKILL with the broker refusing further capability calls, replacing a `Promise.race` that cancelled nothing and let a "timed out" hook go on writing DNS and firewall state (celilo#1003). Capability PROVIDER factories still load in-process — they ARE the broker's implementation. Stages 1 and 2 of `openspec/changes/hook-process-boundary`; the filesystem is claimed by the jail below, and SSH reachability by the remote-ops broker next.
204
+ - **Unjailed advisory lint (task 4.7, NOT a security boundary)** — `apps/celilo/src/hooks/unjailed-lint.ts`. Where there is no jail backend the executor passes the run's derived mount set to the shim in the environment (`CELILO_HOOK_MOUNT_SET`), and the shim wraps the path-taking `node:fs` / `node:fs/promises` functions so an access outside the set (or a write to a read-only row) warns through the hook's own logger: "on a jailed host this would fail", and that it is advisory. It is the mount-set derivation's SECOND consumer, so it cannot drift from what the jail enforces. It observes JS-level `node:fs` calls only — module code bypasses it trivially — and it must never be described as a boundary, in code or output.
205
+ - **Remote-ops broker (reachability scoped by the credential, stage 3 / D12)** — `apps/celilo/src/hooks/remote-broker.ts` (`startRemoteBroker`, `RemoteAccessPolicy`) answering a SECOND socket beside the capability one, with the policy in `apps/celilo/src/services/remote-access.ts` (`remoteAccessPolicy`) and the asking half inside `@celilo/capabilities`' own remote primitives (`packages/capabilities/src/remote.ts`, "The hook remote-ops bridge"). The jail binds no `~/.ssh`, so a hand-built `ssh` cannot authenticate; the primitives detect `CELILO_HOOK_REMOTE_SOCKET`, send each operation as a STRUCTURED request (never a shell string — the broker rebuilds the ssh line itself), and the broker checks the target against `ownedSystemModuleIds` + `getModuleSystems` before running anything, refusing with the module, the target, and the capability route named. Requests carrying the module's OWN credential (an `identityFile` crossing as content and materialised per call, or `installAuthorizedKey`'s password — the cPanel case) are scoped by that credential instead; an explicit non-root user likewise, because the fleet key's authority is root on fleet systems. Stream primitives' LOCAL paths are confined to the run's granted roots (stateDir, screenshots, generated/, declared path inputs), or `streamBackup` would be a write-as-celilo oracle. Attribution is by the module that PERFORMS the operation: a hook's request to the hook's module (the nine `invokeHook` sites build the policy), a provider's transport to the provider's module — providers run in-process and do not cross this socket, and `public_web`'s hand-built upload was replaced by the Ansible static-content converge (capability-owned-tables stage 4, celilo#1014). Residual, recorded rather than papered over: a hook can still `fetch()` any HTTP endpoint directly; only `probeHttp` consults the target check.
206
+ - **Hook jail (a hook sees the paths it was given, and nothing else)** — `apps/celilo/src/hooks/mount-set.ts` (`deriveMountSet`, `toBwrapArgs`, `forbiddenPaths` — PURE, computes a filesystem view and touches nothing) and `apps/celilo/src/hooks/jail.ts` (`detectJailBackend`, `planJailedSpawn`, `realpathRequest`, `runtimeModulePathsFor`, `recordJailMode` — the half that touches the machine). `executeHookScript` spawns the shim under `bwrap` with the module's tree read-only, `state/` + `generated/` + this run's `screenshots/<run>` read-write on top of it, each contract-declared path input at its declared access, the broker's socket directory, the runtime and the `node_modules` the shim resolves through, and `/tmp` a fresh tmpfs FIRST so it cannot erase the socket or a staged input. `~/.ssh` is deliberately NOT bound (stage 3, D12): withholding the credential is what makes the remote-ops broker's target check a boundary. **The acceptance criterion is absence, not a check**: the module store and the data directory are simply not bound, so `master.key` and a sibling module give `ENOENT`. The set is DERIVED — a module cannot ask for more — and `bwrap` itself is never in it, because the AppArmor profile grants `userns` to `/usr/bin/bwrap` for anyone on the box (design D9). Backend detection RUNS bubblewrap rather than looking for it (four different denials all leave the binary in place), and the resulting mode is written to `hook-jail-mode.json` beside celilo's other per-machine state rather than logged, so a host that stops jailing is readable. `CELILO_HOOK_JAIL` = `auto` (default) / `required` / `off`. macOS has a `sandbox-exec` backend in the union, but `auto` defers on it (ce-29z: sandbox-exec DENIES undeclared writes where bubblewrap masks them, and D14's declared-path mechanism is not built), so the hook runs unjailed with the mode recorded; `required` bypasses the deferral for an operator who opts in. An `unjailed` record carries `lastJailed` (the jailed record it replaced on the same host), which is what the `hook_jail` self-monitor reads (`services/alerting/hook-jail.ts`, created unsuppressible by `celilo monitor add hook_jail`) to alert on a host that used to jail and has stopped. `celilo system doctor`'s "Hook execution" section (`renderHookExecutionSection`) reports the live mode and, when unjailed, why.
185
207
  - **Deploy pipeline** — `apps/celilo/src/services/module-deploy.ts`.
186
208
  - **DNS provider backfill** (re-emit registrations when a provider deploys) — `apps/celilo/src/services/dns-provider-backfill.ts` — `isDnsInternalProvider`, `backfillProviderDns`.
187
209
  - **Base-module aspects (fan-out across the fleet)** — `apps/celilo/src/services/aspect-runner.ts` — `planAspectFanOut`, `runAspectFanOut`, `maybeRunAspectForTrigger`. Aspect content lives in `modules/<m>/base-module-aspect/` (e.g. knot-unbound-internal, technitium). Two directions, with DELIBERATELY OPPOSITE failure semantics:
@@ -263,7 +285,7 @@ is currently wrong, and routes carry the message to a person's phone. Design:
263
285
 
264
286
  - **Alert identity** — `apps/celilo/src/services/alerting/keys.ts` — the key grammar (`module:<id>[/check:<name>]`, `builtin:<check>[/<kind>:<target>]`) that makes "the same problem" the same alert across runs. `moduleAlertKey`, `moduleCheckAlertKey`, `builtinAlertKey`, `parseAlertKey`.
265
287
  - **Reconciliation** — `apps/celilo/src/services/alerting/reconcile.ts` (`reconcile`) — a successful run's failing-key set is authoritative and resolution is by SET DIFFERENCE (absent ⇒ resolved). A run whose outcome is `error` resolves NOTHING and fires a module-level alert instead: the false-all-clear guard.
266
- - **Monitor execution** — `apps/celilo/src/services/alerting/run-monitor.ts` (`runOneMonitor`) + `sweep.ts` (`selectDueMonitors`) + `builtin-monitors.ts` / `health-coverage.ts` (the built-in checks and the "module with no health check" coverage check).
288
+ - **Monitor execution** — `apps/celilo/src/services/alerting/run-monitor.ts` (`runOneMonitor`) + `sweep.ts` (`selectDueMonitors`) + `builtin-monitors.ts` / `health-coverage.ts` / `hook-jail.ts` (the built-in checks, the "module with no health check" coverage check, and the hook-jail regression self-monitor).
267
289
  - **Health-check cadence (one accessor)** — `apps/celilo/src/services/alerting/health-cadence.ts` — `effectiveHealthCheckCadence(manifest, override)`, `isScheduled`, `loadModuleHealthCadences`, `reconcileModuleWatchState`. The manifest's `hooks.health_check.interval` SUGGESTS; the operator's `health_check_interval` decides; both resolve at read time. `null` means nobody named a cadence (a coverage gap); `'manual'` means the operator opted out (a decision — raises no coverage finding, and resolves the module's live alerts on the way down, since nothing will report on them again).
268
290
  - **⚠️ `monitors.intervalMinutes` and `monitors.enabled` are `builtin_check`-only.** A `module_hook` row carries severity, escalation policy and `lastRunAt`; its cadence and whether it is watched resolve through the accessor above. The columns' meaning depending on `kind` is a named smell (design.md D8) — the alternatives are a cached resolved value that rots, or splitting the table, which needs a synthetic monitor identity for `alerts.monitorId`. Gates: `sweep-runner.test.ts` asserts a module row's stored values are NOT consulted; `cadence-migration.test.ts` asserts the same through `loadModuleHealthCadences`.
269
291
  - **Carrying an existing fleet over** — `apps/celilo/src/services/alerting/cadence-migration.ts` (`migrateMonitorCadences`), run from `celilo system migrate` (the `.deb` postinst runs it on every apt upgrade). A monitor row whose cadence diverges from its manifest gets that cadence written as an override; a disabled one gets `manual`. Not bookkeeping: without it the upgrade that ships read-time resolution silently reverts every hand-set cadence to the author's suggestion and resumes watching modules an operator deliberately disabled. Idempotent — writes only where no override exists.
package/README.md CHANGED
@@ -286,7 +286,6 @@ bun run test:integration # All integration tests (run via bun test)
286
286
  # - VM resource defaults
287
287
 
288
288
  # What gets skipped:
289
- # - Nix builds
290
289
  # - Docker builds
291
290
  # - Slow module builds (Caddy)
292
291
  ```
@@ -296,7 +295,6 @@ bun run test:integration # All integration tests (run via bun test)
296
295
  bun run test:integration-slow # Full builds + end-to-end
297
296
 
298
297
  # Includes everything from fast tests PLUS:
299
- # - Full Nix builds (Caddy with modules)
300
298
  # - Docker builds
301
299
  # - Complete module packaging
302
300
  ```
@@ -0,0 +1,8 @@
1
+ -- The `environment` column recorded whether a build ran inside a `nix develop`
2
+ -- shell ('nix') or with system tools ('system'). That was the only information
3
+ -- it carried, and the Nix build path never shipped: no module has ever carried
4
+ -- a flake.nix, and the detection code (detectNixEnvironment / isNixAvailable /
5
+ -- the `nix develop` wrapper in module-build.ts) is deleted in the same change.
6
+ -- Every row ever written holds 'system' or NULL, so a constant column: deleted
7
+ -- with the path it described rather than kept as dead weight.
8
+ ALTER TABLE `module_builds` DROP COLUMN `environment`;
@@ -211,6 +211,13 @@
211
211
  "when": 1784400000000,
212
212
  "tag": "0029_module_instances",
213
213
  "breakpoints": true
214
+ },
215
+ {
216
+ "idx": 30,
217
+ "version": "6",
218
+ "when": 1784400000000,
219
+ "tag": "0030_drop_module_builds_environment",
220
+ "breakpoints": true
214
221
  }
215
222
  ]
216
- }
223
+ }
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@celilo/cli",
3
- "version": "1.14.0",
3
+ "version": "2.0.0",
4
4
  "description": "Celilo — home lab orchestration CLI",
5
5
  "type": "module",
6
6
  "bin": {
@@ -58,9 +58,9 @@
58
58
  "dependencies": {
59
59
  "@aws-sdk/client-s3": "^3.1109.0",
60
60
  "@aws-sdk/lib-storage": "^3.1101.0",
61
- "@celilo/capabilities": "^3.4.0",
61
+ "@celilo/capabilities": "^4.0.0",
62
62
  "@celilo/cli-display": "^0.2.0",
63
- "@celilo/core": "^0.10.0",
63
+ "@celilo/core": "^0.11.0",
64
64
  "@celilo/event-bus": "^0.6.0",
65
65
  "ajv": "^8.18.0",
66
66
  "drizzle-orm": "^0.36.4",
@@ -116,6 +116,7 @@ describe('publishStaticSite — clientConfig injection', () => {
116
116
  email: 'admin@example.com',
117
117
  },
118
118
  secrets: {},
119
+ convergeStaticContent: async () => {},
119
120
  routeOps: ops,
120
121
  hostnames: ['www.example.com'],
121
122
  caddyModuleId: 'caddy',
@@ -160,6 +161,7 @@ describe('publishStaticSite — clientConfig injection', () => {
160
161
  email: 'admin@example.com',
161
162
  },
162
163
  secrets: {},
164
+ convergeStaticContent: async () => {},
163
165
  routeOps: ops,
164
166
  hostnames: ['www.example.com'],
165
167
  caddyModuleId: 'caddy',
@@ -187,6 +189,7 @@ describe('publishStaticSite — clientConfig injection', () => {
187
189
  email: 'admin@example.com',
188
190
  },
189
191
  secrets: {},
192
+ convergeStaticContent: async () => {},
190
193
  routeOps: ops,
191
194
  hostnames: ['www.example.com'],
192
195
  caddyModuleId: 'caddy',
@@ -223,6 +226,7 @@ describe('publishStaticSite — clientConfig injection', () => {
223
226
  email: 'admin@example.com',
224
227
  },
225
228
  secrets: {},
229
+ convergeStaticContent: async () => {},
226
230
  routeOps: ops,
227
231
  hostnames: ['www.example.com'],
228
232
  caddyModuleId: 'caddy',
@@ -251,6 +255,7 @@ describe('publishStaticSite — clientConfig injection', () => {
251
255
  email: 'admin@example.com',
252
256
  },
253
257
  secrets: {},
258
+ convergeStaticContent: async () => {},
254
259
  routeOps: ops,
255
260
  hostnames: ['www.example.com'],
256
261
  caddyModuleId: 'caddy',
@@ -289,6 +294,7 @@ describe('registerReverseProxy', () => {
289
294
  email: 'admin@example.com',
290
295
  },
291
296
  secrets: {},
297
+ convergeStaticContent: async () => {},
292
298
  routeOps: ops,
293
299
  hostnames: ['www.example.com'],
294
300
  caddyModuleId: 'caddy',
@@ -325,6 +331,7 @@ describe('registerReverseProxy', () => {
325
331
  email: 'admin@example.com',
326
332
  },
327
333
  secrets: {},
334
+ convergeStaticContent: async () => {},
328
335
  routeOps: ops,
329
336
  hostnames: ['www.example.com'],
330
337
  caddyModuleId: 'caddy',
@@ -372,6 +379,7 @@ describe('auto-logging — end-to-end through createPublicWeb', () => {
372
379
  email: 'admin@example.com',
373
380
  },
374
381
  secrets: {},
382
+ convergeStaticContent: async () => {},
375
383
  routeOps: ops,
376
384
  hostnames: ['www.example.com'],
377
385
  caddyModuleId: 'caddy',
@@ -407,6 +415,7 @@ describe('auto-logging — end-to-end through createPublicWeb', () => {
407
415
  email: 'admin@example.com',
408
416
  },
409
417
  secrets: {},
418
+ convergeStaticContent: async () => {},
410
419
  routeOps: ops,
411
420
  hostnames: ['www.example.com'],
412
421
  caddyModuleId: 'caddy',
@@ -443,6 +452,7 @@ describe('auto-logging — end-to-end through createPublicWeb', () => {
443
452
  email: 'admin@example.com',
444
453
  },
445
454
  secrets: {},
455
+ convergeStaticContent: async () => {},
446
456
  routeOps: ops,
447
457
  hostnames: ['www.example.com'],
448
458
  caddyModuleId: 'caddy',
@@ -479,6 +489,7 @@ describe('auto-logging — end-to-end through createPublicWeb', () => {
479
489
  email: 'admin@example.com',
480
490
  },
481
491
  secrets: {},
492
+ convergeStaticContent: async () => {},
482
493
  routeOps: ops,
483
494
  hostnames: ['www.example.com'],
484
495
  caddyModuleId: 'caddy',
@@ -535,6 +546,7 @@ describe('register_route — public DNS wiring (D1/M1 #328)', () => {
535
546
  logger: noopLogger,
536
547
  config: { target_ip: '10.0.10.20/24' },
537
548
  secrets: {},
549
+ convergeStaticContent: async () => {},
538
550
  routeOps: ops,
539
551
  hostnames: ['www.example.com'],
540
552
  caddyModuleId: 'caddy',
@@ -562,6 +574,7 @@ describe('register_route — public DNS wiring (D1/M1 #328)', () => {
562
574
  logger: noopLogger,
563
575
  config: { target_ip: '10.0.10.20/24' },
564
576
  secrets: {},
577
+ convergeStaticContent: async () => {},
565
578
  routeOps: ops,
566
579
  hostnames: ['www.example.com'],
567
580
  caddyModuleId: 'caddy',
@@ -593,6 +606,7 @@ describe('register_route — public DNS wiring (D1/M1 #328)', () => {
593
606
  logger: noopLogger,
594
607
  config: { target_ip: '10.0.10.20/24' },
595
608
  secrets: {},
609
+ convergeStaticContent: async () => {},
596
610
  routeOps: ops,
597
611
  hostnames: ['www.example.com'],
598
612
  caddyModuleId: 'caddy',
@@ -613,6 +627,7 @@ describe('register_route — public DNS wiring (D1/M1 #328)', () => {
613
627
  logger: noopLogger,
614
628
  config: { target_ip: '10.0.10.20/24' },
615
629
  secrets: {},
630
+ convergeStaticContent: async () => {},
616
631
  routeOps: ops,
617
632
  hostnames: ['www.example.com'],
618
633
  caddyModuleId: 'caddy',
@@ -637,6 +652,7 @@ describe('register_route — public DNS wiring (D1/M1 #328)', () => {
637
652
  logger: noopLogger,
638
653
  config: { target_ip: '10.0.10.20/24' },
639
654
  secrets: {},
655
+ convergeStaticContent: async () => {},
640
656
  routeOps: ops,
641
657
  hostnames: ['www.example.com'],
642
658
  caddyModuleId: 'caddy',
@@ -663,6 +679,7 @@ describe('register_route — public DNS wiring (D1/M1 #328)', () => {
663
679
  // No target_ip and no firewallNatIp → internalDnsIp is falsy.
664
680
  config: {},
665
681
  secrets: {},
682
+ convergeStaticContent: async () => {},
666
683
  routeOps: ops,
667
684
  hostnames: ['www.example.com'],
668
685
  caddyModuleId: 'caddy',
@@ -693,6 +710,7 @@ describe('register_route — public DNS wiring (D1/M1 #328)', () => {
693
710
  logger: noopLogger,
694
711
  config: { target_ip: '10.0.10.20/24' },
695
712
  secrets: {},
713
+ convergeStaticContent: async () => {},
696
714
  routeOps: ops,
697
715
  hostnames: ['www.example.com'],
698
716
  caddyModuleId: 'caddy',
@@ -10,8 +10,10 @@
10
10
  * report counts.
11
11
  */
12
12
 
13
+ import { hostname } from 'node:os';
13
14
  import { getDb } from '../../db/client';
14
15
  import { modules } from '../../db/schema';
16
+ import { readJailMode } from '../../hooks/jail';
15
17
  import { runBuiltinCheckForMonitor } from '../../services/alerting/builtin-source';
16
18
  import { loadModuleCoverage } from '../../services/alerting/coverage-source';
17
19
  import { modulesInDeployWindow } from '../../services/alerting/deploy-hooks';
@@ -61,6 +63,7 @@ export async function handleAlertsSweep(): Promise<CommandResult> {
61
63
  runModuleHealthCheck(moduleId, db, { unattended: true, noInteractive: true }),
62
64
  runBuiltinCheck: (category) => runBuiltinCheckForMonitor(category, db),
63
65
  loadModuleCoverage: () => loadModuleCoverage(db),
66
+ loadJailState: () => ({ record: readJailMode(), host: hostname() }),
64
67
  now: () => new Date(),
65
68
  graceMs: DEFAULT_GRACE_MS,
66
69
  },
@@ -6,10 +6,12 @@
6
6
  * and audit checks into the injectable deps `runOneMonitor` expects.
7
7
  */
8
8
 
9
+ import { hostname } from 'node:os';
9
10
  import { defineEvents, openBus } from '@celilo/event-bus';
10
11
  import { getEventBusPath } from '../../config/paths';
11
12
  import { getDb } from '../../db/client';
12
13
  import type { MonitorKind } from '../../db/schema';
14
+ import { readJailMode } from '../../hooks/jail';
13
15
  import {
14
16
  isSchedulableBuiltin,
15
17
  runBuiltinCheckForMonitor,
@@ -17,6 +19,7 @@ import {
17
19
  import { loadModuleCoverage } from '../../services/alerting/coverage-source';
18
20
  import { loadModuleHealthCadences } from '../../services/alerting/health-cadence';
19
21
  import { HEALTH_COVERAGE_CHECK } from '../../services/alerting/health-coverage';
22
+ import { HOOK_JAIL_CHECK } from '../../services/alerting/hook-jail';
20
23
  import {
21
24
  createMonitor,
22
25
  ensureSweepSubscriber,
@@ -50,6 +53,7 @@ function buildDeps() {
50
53
  runModuleHealthCheck(moduleId, db, { unattended: true, noInteractive: true }),
51
54
  runBuiltinCheck: (category: DriftCategory) => runBuiltinCheckForMonitor(category, db),
52
55
  loadModuleCoverage: () => loadModuleCoverage(db),
56
+ loadJailState: () => ({ record: readJailMode(), host: hostname() }),
53
57
  now: () => new Date(),
54
58
  graceMs: DEFAULT_GRACE_MS,
55
59
  };
@@ -120,7 +124,10 @@ function handleAdd(args: string[], flags: Record<string, boolean | string>): Com
120
124
  // underscore heuristic alone would file it as a module hook against a module
121
125
  // that does not exist.
122
126
  const kind: MonitorKind =
123
- target === HEALTH_COVERAGE_CHECK || isSchedulableBuiltin(target) || target.includes('_')
127
+ target === HEALTH_COVERAGE_CHECK ||
128
+ target === HOOK_JAIL_CHECK ||
129
+ isSchedulableBuiltin(target) ||
130
+ target.includes('_')
124
131
  ? 'builtin_check'
125
132
  : 'module_hook';
126
133
 
@@ -148,7 +155,13 @@ function handleAdd(args: string[], flags: Record<string, boolean | string>): Com
148
155
  const cadence = parseCadence(interval);
149
156
  const intervalMinutes = cadence !== null && cadence !== 'manual' ? cadence.minutes : 0;
150
157
 
151
- createMonitor(db, { kind, target, intervalMinutes });
158
+ // The hook-jail check is a SELF-monitor: it watches celilo's own confinement
159
+ // of module hooks, and the alerting spec marks monitors watching the system
160
+ // itself unsuppressible. That is a property of the target, not an operator
161
+ // choice — losing the jail during a broad outage is exactly when no ancestor
162
+ // alert may silence it.
163
+ const suppressible = target === HOOK_JAIL_CHECK ? false : undefined;
164
+ createMonitor(db, { kind, target, intervalMinutes, suppressible });
152
165
 
153
166
  // Creating the first monitor is also what switches the sweep on. Registering
154
167
  // here rather than at install means a celilo with no monitors carries no
@@ -1,5 +1,10 @@
1
1
  import { describe, expect, test } from 'bun:test';
2
- import { compareVersions } from './system-doctor';
2
+ import type { JailModeRecord } from '../../hooks/jail';
3
+ import {
4
+ type HookExecutionInput,
5
+ compareVersions,
6
+ renderHookExecutionSection,
7
+ } from './system-doctor';
3
8
 
4
9
  describe('compareVersions', () => {
5
10
  test('detects ascending major/minor/patch', () => {
@@ -34,3 +39,118 @@ describe('compareVersions', () => {
34
39
  expect(compareVersions('v1.2.3', '1.2.3')).toBe(0);
35
40
  });
36
41
  });
42
+
43
+ describe('renderHookExecutionSection (hook-process-boundary task 4.6)', () => {
44
+ const HOST = 'celilo-mgr';
45
+ const jailedRecord: JailModeRecord = {
46
+ mode: 'jailed',
47
+ backend: 'bubblewrap',
48
+ host: HOST,
49
+ recordedAt: '2026-08-28T09:00:00.000Z',
50
+ };
51
+ const regressedRecord: JailModeRecord = {
52
+ mode: 'unjailed',
53
+ backend: 'none',
54
+ reason: 'bubblewrap is installed but could not build a namespace, so hooks run unjailed.',
55
+ host: HOST,
56
+ recordedAt: '2026-08-28T09:00:00.000Z',
57
+ lastJailed: { backend: 'bubblewrap', recordedAt: '2026-08-27T09:00:00.000Z' },
58
+ };
59
+
60
+ const render = (over: Partial<HookExecutionInput>) =>
61
+ renderHookExecutionSection({
62
+ availability: { backend: 'bubblewrap' },
63
+ policy: 'auto',
64
+ record: jailedRecord,
65
+ host: HOST,
66
+ monitored: true,
67
+ ...over,
68
+ });
69
+
70
+ const text = (r: { lines: string[] }) => r.lines.join('\n');
71
+
72
+ test('a jailing host is a ✔ and nothing more', () => {
73
+ const r = render({});
74
+ expect(text(r)).toContain('hooks run jailed (bubblewrap)');
75
+ expect(r.failCount).toBe(0);
76
+ expect(r.warnCount).toBe(0);
77
+ });
78
+
79
+ test('unjailed says why in the reason sentence, as a warning and not a failure', () => {
80
+ const r = render({
81
+ availability: {
82
+ backend: 'none',
83
+ reason:
84
+ 'bubblewrap is not installed, so hooks run unjailed. Install it (`apt install bubblewrap`) and re-run.',
85
+ },
86
+ record: undefined,
87
+ monitored: null,
88
+ });
89
+ expect(text(r)).toContain('hooks run unjailed — bubblewrap is not installed');
90
+ expect(text(r)).toContain('no execution mode recorded yet');
91
+ expect(r.warnCount).toBe(1);
92
+ expect(r.failCount).toBe(0);
93
+ });
94
+
95
+ test('CELILO_HOOK_JAIL=off is reported as the operator’s own act', () => {
96
+ const r = render({ policy: 'off', record: undefined });
97
+ expect(text(r)).toContain('CELILO_HOOK_JAIL=off');
98
+ expect(r.warnCount).toBe(1);
99
+ });
100
+
101
+ test('auto deferral on sandbox-exec reads as unjailed, not jailed (ce-29z)', () => {
102
+ // The doctor and the executor share one predicate (autoJailDefers), so
103
+ // the doctor cannot report a jail the executor will not build. When D14
104
+ // lands and the flag flips, this goes back to ✔ with no code change here.
105
+ const r = render({ availability: { backend: 'sandbox-exec' }, record: undefined });
106
+ expect(text(r)).toContain('hooks run unjailed — macOS hooks run unjailed');
107
+ expect(text(r)).not.toContain('hooks run jailed');
108
+ expect(r.warnCount).toBe(1);
109
+ expect(r.failCount).toBe(0);
110
+ });
111
+
112
+ test('required with no backend is a failure, because every hook fails', () => {
113
+ const r = render({
114
+ policy: 'required',
115
+ availability: { backend: 'none', reason: 'the profile did not load' },
116
+ record: undefined,
117
+ });
118
+ expect(text(r)).toContain('CELILO_HOOK_JAIL=required and no jail is available');
119
+ expect(r.failCount).toBe(1);
120
+ });
121
+
122
+ test('a host that used to jail and has stopped is a failure naming when', () => {
123
+ const r = render({
124
+ availability: { backend: 'none', reason: 'the profile did not load' },
125
+ record: regressedRecord,
126
+ });
127
+ expect(text(r)).toContain('ran hooks jailed (bubblewrap) until 2026-08-27T09:00:00.000Z');
128
+ expect(r.failCount).toBe(1);
129
+ });
130
+
131
+ test('an unmonitored regression points at the self-monitor', () => {
132
+ const r = render({
133
+ availability: { backend: 'none', reason: 'the profile did not load' },
134
+ record: regressedRecord,
135
+ monitored: false,
136
+ });
137
+ expect(text(r)).toContain('celilo monitor add hook_jail');
138
+ });
139
+
140
+ test('a jailing but unmonitored host is told the transition alerts only if monitored', () => {
141
+ const r = render({ monitored: false });
142
+ expect(text(r)).toContain('celilo monitor add hook_jail');
143
+ expect(r.failCount).toBe(0);
144
+ });
145
+
146
+ test('a dev box with no DB gets no monitor hint', () => {
147
+ const r = render({ monitored: null });
148
+ expect(text(r)).not.toContain('monitor add');
149
+ });
150
+
151
+ test("another host's record reads as a move, not this host's history", () => {
152
+ const r = render({ record: { ...regressedRecord, host: 'old-box' } });
153
+ expect(text(r)).toContain('belongs to host "old-box"');
154
+ expect(r.failCount).toBe(0);
155
+ });
156
+ });