@celilo/cli 0.8.1 → 0.9.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/AGENTS.md +86 -0
- package/CELILO_CORE_MODULES.md +61 -0
- package/CELILO_SUBSYSTEMS.md +83 -0
- package/drizzle/0012_module_systems_sizing.sql +3 -0
- package/drizzle/0013_dns_view_overrides.sql +1 -0
- package/drizzle/meta/_journal.json +14 -0
- package/package.json +6 -3
- package/src/capabilities/well-known.test.ts +12 -7
- package/src/capabilities/well-known.ts +11 -3
- package/src/cli/command-registry.ts +65 -1
- package/src/cli/commands/module-upgrade.test.ts +29 -0
- package/src/cli/commands/module-upgrade.ts +57 -24
- package/src/cli/commands/proxmox-instance-list.test.ts +77 -0
- package/src/cli/commands/proxmox-instance-list.ts +140 -0
- package/src/cli/commands/proxmox-instance-resize.ts +235 -0
- package/src/cli/commands/proxmox-node-list.ts +1 -34
- package/src/cli/commands/proxmox-resize-guards.test.ts +55 -0
- package/src/cli/commands/proxmox-resize-guards.ts +102 -0
- package/src/cli/commands/proxmox-service.ts +38 -0
- package/src/cli/completion.ts +11 -3
- package/src/cli/index.ts +15 -0
- package/src/db/schema.ts +21 -1
- package/src/hooks/capability-loader.ts +22 -0
- package/src/manifest/template-validator.test.ts +31 -1
- package/src/manifest/template-validator.ts +9 -0
- package/src/services/deployed-systems.test.ts +73 -1
- package/src/services/deployed-systems.ts +72 -0
- package/src/services/dns-internal-records.test.ts +76 -3
- package/src/services/dns-internal-records.ts +52 -3
- package/src/services/dns-provider-backfill.ts +15 -3
- package/src/services/fleet-checks.test.ts +18 -16
- package/src/services/machine-detector.ts +34 -12
- package/src/templates/generator.ts +49 -1
- package/src/variables/context.ts +36 -7
package/AGENTS.md
ADDED
|
@@ -0,0 +1,86 @@
|
|
|
1
|
+
# AGENTS.md — `@celilo/cli`
|
|
2
|
+
|
|
3
|
+
Celilo is a **home-lab orchestration system**: you describe infrastructure as
|
|
4
|
+
**modules**, and celilo deploys them onto a fleet (Proxmox LXC/VM) and wires them
|
|
5
|
+
together through versioned **capabilities**. This file orients an AI agent to
|
|
6
|
+
build and ship apps on celilo. **Read the local docs below before searching the
|
|
7
|
+
web** — they are the source of truth; the hosted copies at
|
|
8
|
+
`https://celilo.computer/docs` lag them.
|
|
9
|
+
|
|
10
|
+
## Mental model (learn these five things)
|
|
11
|
+
|
|
12
|
+
- **Module** — a directory with a `manifest.yml` and `celilo/scripts/` hooks,
|
|
13
|
+
packaged into a `.netapp` artifact and published to **celilo-registry**.
|
|
14
|
+
Everything celilo deploys is a module.
|
|
15
|
+
- **Capability** — a module `provides:` capabilities (e.g. `public_web`, `idp`,
|
|
16
|
+
`source_forge`) at a **versioned contract**, and consumers `requires:` them.
|
|
17
|
+
Consumers bind to the **capability** version (`provides.capabilities[].version`)
|
|
18
|
+
— *not* the module's own `version`. Bump the capability version on a contract
|
|
19
|
+
change; that's the load-bearing semver.
|
|
20
|
+
- **Hook** — a lifecycle script (`on_install`, `health_check`, `on_backup`, …) in
|
|
21
|
+
`celilo/scripts/`, written with `defineHook(...)` from `@celilo/capabilities`.
|
|
22
|
+
Hooks receive typed clients for the capabilities they `requires`/`optional`.
|
|
23
|
+
- **Build bus** — app repo CI publishes a `.netapp` to celilo-registry →
|
|
24
|
+
celilo-mgr **polls** → `module upgrade` → deploy → verify. No manual hop.
|
|
25
|
+
- **Versioning** — `manifest.yml#version` is the **payload** version; `+N` is the
|
|
26
|
+
recipe revision; `version_source` declares how the version is set. See below.
|
|
27
|
+
|
|
28
|
+
## Building an app (the happy path)
|
|
29
|
+
|
|
30
|
+
An **app** is a module that provides no capability (it serves content), so set
|
|
31
|
+
`version_source: { kind: changeset }`.
|
|
32
|
+
|
|
33
|
+
1. **`manifest.yml`** — `id` (kebab-case), `celilo_contract: "1.0"`, `version`,
|
|
34
|
+
the capabilities you `requires:` (e.g. `public_web`, `idp`), `variables`,
|
|
35
|
+
`hooks`, and `version_source`. Keep the schema directive on line 1 so your
|
|
36
|
+
editor validates:
|
|
37
|
+
`# yaml-language-server: $schema=../../schemas/module-manifest.schema.json`
|
|
38
|
+
2. **Hooks** in `celilo/scripts/` — at least `on_install` (provision) and
|
|
39
|
+
`health_check`. Use `defineHook`; required capabilities are typed non-null.
|
|
40
|
+
3. **CI/CD** — copy the reference workflows + add the `register-forge` deploy
|
|
41
|
+
hook so your repo gets a scoped publish token (see `APP_CI_REFERENCE.md`).
|
|
42
|
+
4. **Version with changesets** — `celilo module changeset --bump <major|minor|
|
|
43
|
+
patch>` in each behaviour-changing PR; `celilo module version` stamps
|
|
44
|
+
`manifest.yml#version` + CHANGELOG at release. Pick the bump by deploy **blast
|
|
45
|
+
radius** (major ⇒ safe deploy w/ backup; patch ⇒ fast).
|
|
46
|
+
5. **Ship** — `celilo module check .` then `celilo module publish .`. On the
|
|
47
|
+
fleet: `module update` (registry sweep) → `module deploy <id>` → `module
|
|
48
|
+
health`.
|
|
49
|
+
|
|
50
|
+
## CLI you'll use
|
|
51
|
+
|
|
52
|
+
```
|
|
53
|
+
celilo module import <name|path> # pull a module
|
|
54
|
+
celilo module check [dir] # validate: schema, capabilities, git-hygiene
|
|
55
|
+
celilo module changeset --bump <b> -m … # author a version changeset (module-id keyed)
|
|
56
|
+
celilo module version [dir] # stamp version+CHANGELOG from changesets
|
|
57
|
+
celilo module publish [dir] # build .netapp + publish to celilo-registry
|
|
58
|
+
celilo module deploy <id> / update / health # deploy + verify on the fleet
|
|
59
|
+
```
|
|
60
|
+
|
|
61
|
+
## Conventions / rules
|
|
62
|
+
|
|
63
|
+
- Module `id` is **kebab-case**; pin `celilo_contract: "1.0"`.
|
|
64
|
+
- **`version_source`** picks how `version` is sourced: `changeset` (apps/content —
|
|
65
|
+
authored via changesets, ordered by `+N`), `pin` (wraps upstream software —
|
|
66
|
+
`version` = the installed upstream version, resolver-checked), `recipe`
|
|
67
|
+
(config-only modules — no payload version; `+N` orders). Default `recipe`.
|
|
68
|
+
- The capability **contract** semver lives in `provides.capabilities[].version`,
|
|
69
|
+
independent of the module `version`. Don't conflate them.
|
|
70
|
+
- **Never hand-edit `schemas/module-manifest.schema.json`** — it's generated from
|
|
71
|
+
`src/manifest/schema.ts` via `bun run export:schema`.
|
|
72
|
+
- Validate before publishing; `celilo module check` mirrors the publish gates.
|
|
73
|
+
|
|
74
|
+
## Doc map (read in this order)
|
|
75
|
+
|
|
76
|
+
- `./CELILO_SUBSYSTEMS.md` — map of the infrastructure primitives celilo already
|
|
77
|
+
implements (IPAM, capabilities, firewall/DNS, generator, …), with entry-point files.
|
|
78
|
+
- `./CELILO_CORE_MODULES.md` — the production modules celilo ships in `modules/`
|
|
79
|
+
(what each provides/requires, its role in the fleet).
|
|
80
|
+
- `../../design/MODULE_DEVELOPMENT_GUIDE.md` — author a module (start here).
|
|
81
|
+
- `./CLI_USAGE.md` — full CLI reference + workflows.
|
|
82
|
+
- `../../v2/BUILD_BUS.md` — how a change reaches the fleet.
|
|
83
|
+
- `../../v2/APP_CI_REFERENCE.md` — wire an app's CI/CD (the copy-me recipe).
|
|
84
|
+
- `../../v2/MODULE_VERSIONING.md` — what `version` means; `version_source`.
|
|
85
|
+
- `../../schemas/module-manifest.schema.json` — the manifest contract.
|
|
86
|
+
- `https://celilo.computer/docs` — hosted docs (LAN; the repo docs lead).
|
|
@@ -0,0 +1,61 @@
|
|
|
1
|
+
# Celilo Core Modules
|
|
2
|
+
|
|
3
|
+
**What this is:** the production modules celilo ships in `modules/` — the infrastructure celilo
|
|
4
|
+
actually *deploys*. Companion to [CELILO_SUBSYSTEMS.md](./CELILO_SUBSYSTEMS.md) (which maps
|
|
5
|
+
celilo's own engine). Use this to know what already exists before building a new module, and to
|
|
6
|
+
understand how the fleet wires together via capabilities.
|
|
7
|
+
|
|
8
|
+
Each entry: `module id` — what it is — **provides** / **requires** capabilities. Its source is
|
|
9
|
+
`modules/<id>/manifest.yml`; capability contracts are listed in
|
|
10
|
+
[CELILO_SUBSYSTEMS.md](./CELILO_SUBSYSTEMS.md).
|
|
11
|
+
|
|
12
|
+
> **Paths below are relative to the celilo source-repo root** (e.g. `modules/caddy/manifest.yml`)
|
|
13
|
+
> — references into the source tree, not links relative to this file (this doc also ships inside
|
|
14
|
+
> the `@celilo/cli` npm package). Re-grep manifests if an entry looks stale.
|
|
15
|
+
|
|
16
|
+
> How the graph closes: a public app (`requires: public_web`) is served by **caddy**, which
|
|
17
|
+
> `requires: dns_registrar` (**namecheap**) to publish its name and `firewall` (**greenwave** /
|
|
18
|
+
> **iptables**) to open the port. Internal name resolution comes from a **dns_internal** provider
|
|
19
|
+
> (**knot-unbound-internal** / **technitium**). Identity comes from **authentik** (`idp`). That's
|
|
20
|
+
> the whole edge: DNS + firewall + ingress + identity, each a swappable provider module.
|
|
21
|
+
|
|
22
|
+
## Network fabric (DNS / firewall / DHCP)
|
|
23
|
+
|
|
24
|
+
- **greenwave** — GreenWave C4000XG ISP router driver; port-forwarding + public-IP discovery via REST. **provides:** `firewall`, `dhcp_server`.
|
|
25
|
+
- **iptables** — iptables firewall + NAT; cross-VLAN port exposure with recursive upstream delegation. **provides:** `firewall`.
|
|
26
|
+
- **knot-unbound-internal** — split-horizon internal DNS via Knot (authoritative) + Unbound (recursive); lightweight, plain apt, no .NET. **provides:** `dns_internal`. Ships a base-module-aspect (`modules/knot-unbound-internal/base-module-aspect/`).
|
|
27
|
+
- **technitium** — internal split-horizon DNS resolver + authoritative server (web UI + HTTP API); heavier alternative to knot-unbound. **provides:** `dns_internal`. Ships a base-module-aspect (`modules/technitium/base-module-aspect/`).
|
|
28
|
+
- **namecheap** — public DNS A-record management via Namecheap Dynamic DNS API (HTTP, no browser automation). **provides:** `dns_registrar`.
|
|
29
|
+
|
|
30
|
+
## Public edge (ingress / identity)
|
|
31
|
+
|
|
32
|
+
- **caddy** — reverse proxy with automatic HTTPS (HTTP-01 ACME); the standard HTTPS ingress. **provides:** `public_web`. **requires:** `dns_registrar`, `firewall`.
|
|
33
|
+
- **authentik** — Authentik identity provider with OIDC (Docker Compose: server, worker, Postgres, Redis). **provides:** `idp`. **requires:** `public_web`, `dns_registrar`, `firewall`.
|
|
34
|
+
|
|
35
|
+
## Celilo's own infrastructure (self-hosted)
|
|
36
|
+
|
|
37
|
+
- **celilo-mgmt** — the celilo management server itself, deployed as a module (replaces install.sh + `system init`; ships daemon, runs migrations, self-registers). **provides:** `celilo_event_bus`, `celilo_module_deploy_worker`. **requires:** `cross_module_read`. See `v2/MANAGEMENT_AS_NETAPP.md`.
|
|
38
|
+
- **celilo-registry** — module registry server (Cargo sparse protocol); stores `.netapp` files, serves index + search/download API. **provides:** `registry_publish`. **requires:** `public_web`, `dns_registrar`.
|
|
39
|
+
- **celilo-apt-repo** — Debian apt repository (reprepro + Bun HTTP server) serving the celilo `.deb` at apt.celilo.computer. **provides:** `apt_publish`. **requires:** `public_web`, `dns_registrar`.
|
|
40
|
+
- **celilo-website** — public docs site (static Astro) served via Caddy on celilo.computer. **requires:** `public_web`, `dns_registrar`.
|
|
41
|
+
|
|
42
|
+
## Git forge & CI pipeline
|
|
43
|
+
|
|
44
|
+
- **forgejo** — self-hosted Forgejo git forge (git-over-SSH, OIDC, public ingress). **provides:** `source_forge`. **requires:** `public_web`, `idp`, `firewall`, `dns_registrar`. See `v2/CI_PIPELINE.md`.
|
|
45
|
+
- **forgejo-runner** — host-mode (LXC) Forgejo Actions runner for unit/lint jobs; outbound-only, lives in dmz. **requires:** `source_forge`. See `v2/FORGEJO_RUNNER.md`.
|
|
46
|
+
- **forgejo-builder** — VM-based, Docker-capable Forgejo Actions runner (`requires.system.type: vm`) for hermetic release builds + the cele2e suite. **requires:** `source_forge`.
|
|
47
|
+
- **npm-cache-node** — self-hosted npm registry as a pass-through disk cache (proxies upstream). **requires:** `public_web`, `dns_registrar`. See `v2/NPM_CACHE_NODE.md`.
|
|
48
|
+
|
|
49
|
+
## Applications
|
|
50
|
+
|
|
51
|
+
- **homebridge** — HomeKit bridge for smart-home devices (VeSync, Leviton, Lutron, TP-Link, Tuya). No capabilities (leaf app).
|
|
52
|
+
|
|
53
|
+
## E2E fixtures & probes (not production apps)
|
|
54
|
+
|
|
55
|
+
- **source-forge-probe** — hostless e2e probe that exercises the `source_forge` capability (issue/revoke runner token) against a deployed Forgejo. **requires:** `source_forge`.
|
|
56
|
+
- **test-ansible-output** — e2e fixture that runs 25×1s sleep tasks to assert Ansible output streams in real time. Not a real app.
|
|
57
|
+
- **vm-smoke** — throwaway module proving celilo's generic `type: vm` deploy path end-to-end (template clone → cloud-init → Ansible reach). No capabilities.
|
|
58
|
+
|
|
59
|
+
## Archived / superseded
|
|
60
|
+
|
|
61
|
+
`modules/archive/` holds retired modules — **dns-external** (VPS authoritative DNS + WireGuard, superseded by the `dns_internal`/`dns_registrar` split), **gmail** (email-reading capability), **namecheap-api** (registrar-config via Namecheap API, superseded by the **namecheap** DDNS module). Reference only; not deployed.
|
|
@@ -0,0 +1,83 @@
|
|
|
1
|
+
# Celilo Subsystem Inventory
|
|
2
|
+
|
|
3
|
+
**What this is:** a map of infrastructure primitives celilo **already implements**, with the
|
|
4
|
+
entry-point file for each. It exists so agents stop reinventing features (the trigger case:
|
|
5
|
+
IP/VMID allocation, which is already IPAM).
|
|
6
|
+
|
|
7
|
+
**The rule:** Assume celilo already provides any infrastructure primitive (IP/VMID
|
|
8
|
+
allocation, secrets, DNS, firewall, cross-module data, packaging, event scheduling).
|
|
9
|
+
Before proposing to build one — or asking the operator how to do it — grep this
|
|
10
|
+
inventory / the codebase and cite what you found. Only raise a question if the search
|
|
11
|
+
is genuinely empty, or the real choice is between existing mechanisms.
|
|
12
|
+
|
|
13
|
+
This is an *implemented-feature* map. For terminology see `GLOSSARY.md`; for architecture
|
|
14
|
+
see `design/README.md`. Companion doc: [CELILO_CORE_MODULES.md](./CELILO_CORE_MODULES.md)
|
|
15
|
+
(the production modules celilo ships).
|
|
16
|
+
|
|
17
|
+
> **All file paths below are relative to the celilo source-repo root** (e.g.
|
|
18
|
+
> `apps/celilo/src/ipam/allocator.ts`). They are references into the source tree, not links
|
|
19
|
+
> relative to this file — this doc also ships inside the `@celilo/cli` npm package, where the
|
|
20
|
+
> wider repo isn't present. Paths drift; if an entry looks wrong, re-grep and fix it here.
|
|
21
|
+
|
|
22
|
+
---
|
|
23
|
+
|
|
24
|
+
## Allocation & infrastructure selection
|
|
25
|
+
|
|
26
|
+
- **IPAM (IP/VMID allocation)** — `apps/celilo/src/ipam/allocator.ts` — `allocateIPFromSubnet`, `allocateVMID`, `reserveIP`/`unreserveIP`, `inferZoneFromIP`, `getAllocation`. Auto-wrapper: `apps/celilo/src/ipam/auto-allocator.ts` — `allocateForModule` / `deallocateForModule`.
|
|
27
|
+
- **Infrastructure selection (container-service vs machine pool)** — `apps/celilo/src/services/machine-pool.ts` (`getMachineByHostname`, `addMachine`, `assignModuleToMachine`) and `apps/celilo/src/services/container-service.ts` (`getContainerServiceByName`, `addContainerService`, `verifyContainerService`). Provider API clients: `apps/celilo/src/api-clients/proxmox.ts`, `apps/celilo/src/api-clients/digitalocean.ts`.
|
|
28
|
+
- **Zone detection / system config** — `apps/celilo/src/services/zone-detector.ts` — `detectZoneFromIp` reads `network.<zone>.subnet` from the `systemConfig` table.
|
|
29
|
+
|
|
30
|
+
## Capability system (cross-module data & functions)
|
|
31
|
+
|
|
32
|
+
- **Define a capability function** — `packages/capabilities/src/define-capability-function.ts` — `defineCapabilityFunction`.
|
|
33
|
+
- **Canonical capability registry** — `packages/capabilities/src/capability-registry.ts` — `KNOWN_CAPABILITY_NAMES` (the authoritative list), `CapabilityRegistry` type. Public surface: `packages/capabilities/src/index.ts`.
|
|
34
|
+
- **Loader (wires provider factories into hook contexts)** — `apps/celilo/src/hooks/capability-loader.ts` — `loadCapabilityFunctions`, `resolveFirewallNatIp`.
|
|
35
|
+
- **Ledger wrappers (stateful capabilities)** — `apps/celilo/src/services/dns-registrations.ts` (`withDnsRegistrationLedger`), `apps/celilo/src/services/dns-internal-records.ts` (`withDnsInternalLedger`).
|
|
36
|
+
|
|
37
|
+
### Known capabilities (impl → provider module)
|
|
38
|
+
|
|
39
|
+
| capability | impl | provider module(s) |
|
|
40
|
+
|---|---|---|
|
|
41
|
+
| `public_web` | `packages/capabilities/src/public-web.ts` (`createPublicWeb`) | caddy |
|
|
42
|
+
| `idp` | `packages/capabilities/src/idp.ts` | authentik |
|
|
43
|
+
| `dns_registrar` | `packages/capabilities/src/dns-registrar.ts` (`registerHost`) | namecheap |
|
|
44
|
+
| `firewall` | `packages/capabilities/src/firewall.ts` (`exposeService`, `unexposeService`, `listExposedServices`) | greenwave, iptables |
|
|
45
|
+
| `dns_internal` | `packages/capabilities/src/dns-internal.ts` | knot-unbound-internal, technitium |
|
|
46
|
+
| `dhcp_server` | `packages/capabilities/src/dhcp-server.ts` | greenwave |
|
|
47
|
+
| `source_forge` | `packages/capabilities/src/source-forge.ts` | forgejo |
|
|
48
|
+
| `registry_publish` | `packages/capabilities/src/registry-publish.ts` | celilo-registry |
|
|
49
|
+
| `cross_module_read` | `packages/capabilities/src/cross-module-read.ts` | framework (read other modules' capability data) |
|
|
50
|
+
|
|
51
|
+
## Hooks & deploy
|
|
52
|
+
|
|
53
|
+
- **Hook executor / ABI** — `apps/celilo/src/hooks/executor.ts` (`invokeHook`, `executeHookScript`, `checkRequiredCapabilities`), types in `apps/celilo/src/hooks/types.ts` (`HookContext`, `HookDefinition`, `HookName`). Named-hook runner: `apps/celilo/src/hooks/run-named-hook.ts`. Manifest hook config: `apps/celilo/src/hooks/load-hook-config.ts`.
|
|
54
|
+
- **Deploy pipeline** — `apps/celilo/src/services/module-deploy.ts`.
|
|
55
|
+
- **DNS provider backfill** (re-emit registrations when a provider deploys) — `apps/celilo/src/services/dns-provider-backfill.ts` — `isDnsInternalProvider`, `backfillProviderDns`.
|
|
56
|
+
- **Base-module aspects (fan-out across the fleet)** — `apps/celilo/src/services/aspect-runner.ts` — `planAspectFanOut`, `runAspectFanOut`, `maybeRunAspectForTrigger`. Aspect content lives in `modules/<m>/base-module-aspect/` (e.g. knot-unbound-internal, technitium).
|
|
57
|
+
|
|
58
|
+
## Generation & templating
|
|
59
|
+
|
|
60
|
+
- **Generator** — `apps/celilo/src/templates/generator.ts` — `generateTemplates` (orchestration), plus Terraform/Ansible file handling.
|
|
61
|
+
- **Variable resolution** — `apps/celilo/src/variables/resolver.ts` (parser: `apps/celilo/src/variables/parser.ts`). Supported prefixes: `$self`, `$system`, `$secret`, `$system_secret`, `$capability`, `$infra`.
|
|
62
|
+
|
|
63
|
+
## Secrets
|
|
64
|
+
|
|
65
|
+
- **Vault / encryption** — `apps/celilo/src/secrets/vault.ts` (`deriveVaultPassword`, `getVaultPassword` — Ansible Vault) and `apps/celilo/src/secrets/encryption.ts` (`encryptSecret`/`decryptSecret`, AES-256-GCM).
|
|
66
|
+
|
|
67
|
+
## Packaging, registry & publish
|
|
68
|
+
|
|
69
|
+
- **Module packaging** — `apps/celilo/src/module/packaging/` — `build.ts` (`buildModule`), `extract.ts`, `checksum.ts`, `signature.ts` (`signChecksums`/`verifySignature`), `release-metadata.ts`, `audit.ts`.
|
|
70
|
+
- **Publish driver** — `scripts/publish.ts` shims to `apps/celilo/src/cli/commands/publish/` (workspace npm packages via `bun publish` + module registry; preflight stale-version/stale-manifest gates).
|
|
71
|
+
|
|
72
|
+
## Persistence
|
|
73
|
+
|
|
74
|
+
- **DB schema** — `apps/celilo/src/db/schema.ts`. Client: `apps/celilo/src/db/client.ts`. Migration runner: `apps/celilo/src/db/migrate.ts`. Migrations: `apps/celilo/drizzle/`.
|
|
75
|
+
|
|
76
|
+
## Events
|
|
77
|
+
|
|
78
|
+
- **Event bus** — `packages/event-bus/src/index.ts` — `Bus`, `openBus`, `defineEvents`, `defineHandler`, `runDispatcher`, pattern matching + timer ticks (`emitDueTimerTicks`, `retentionSweep`).
|
|
79
|
+
|
|
80
|
+
## E2E simulation
|
|
81
|
+
|
|
82
|
+
- **cele2e harness** — `packages/e2e/src/` — `runner.ts`, `container-manager.ts` (`startNetwork`, `reconnectNetwork`), `network-builder.ts` (`NetworkBuilder`).
|
|
83
|
+
- **Run wrapper** — `infra/scripts/cele2e-run.sh` lives in the separate `infra/` clone, **not** in this repo. See the cele2e section of the repo-root `CLAUDE.md` for the operator workflow.
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
ALTER TABLE `dns_internal_records` ADD `zone_routable_ip` text;
|
|
@@ -85,6 +85,20 @@
|
|
|
85
85
|
"when": 1781481660000,
|
|
86
86
|
"tag": "0011_backups_name",
|
|
87
87
|
"breakpoints": true
|
|
88
|
+
},
|
|
89
|
+
{
|
|
90
|
+
"idx": 12,
|
|
91
|
+
"version": "6",
|
|
92
|
+
"when": 1781481720000,
|
|
93
|
+
"tag": "0012_module_systems_sizing",
|
|
94
|
+
"breakpoints": true
|
|
95
|
+
},
|
|
96
|
+
{
|
|
97
|
+
"idx": 13,
|
|
98
|
+
"version": "6",
|
|
99
|
+
"when": 1782456724000,
|
|
100
|
+
"tag": "0013_dns_view_overrides",
|
|
101
|
+
"breakpoints": true
|
|
88
102
|
}
|
|
89
103
|
]
|
|
90
104
|
}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@celilo/cli",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.9.0",
|
|
4
4
|
"description": "Celilo — home lab orchestration CLI",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"bin": {
|
|
@@ -11,7 +11,10 @@
|
|
|
11
11
|
"src/",
|
|
12
12
|
"drizzle/",
|
|
13
13
|
"schemas/",
|
|
14
|
-
"tsconfig.json"
|
|
14
|
+
"tsconfig.json",
|
|
15
|
+
"AGENTS.md",
|
|
16
|
+
"CELILO_SUBSYSTEMS.md",
|
|
17
|
+
"CELILO_CORE_MODULES.md"
|
|
15
18
|
],
|
|
16
19
|
"keywords": [
|
|
17
20
|
"celilo",
|
|
@@ -52,7 +55,7 @@
|
|
|
52
55
|
},
|
|
53
56
|
"dependencies": {
|
|
54
57
|
"@aws-sdk/client-s3": "^3.1024.0",
|
|
55
|
-
"@celilo/capabilities": "^0.
|
|
58
|
+
"@celilo/capabilities": "^0.6.0",
|
|
56
59
|
"@celilo/cli-display": "^0.1.9",
|
|
57
60
|
"@celilo/event-bus": "^0.1.7",
|
|
58
61
|
"@clack/prompts": "^1.1.0",
|
|
@@ -27,7 +27,9 @@ describe('Well-Known Capabilities Registry', () => {
|
|
|
27
27
|
test('should have dns_internal capability', () => {
|
|
28
28
|
expect(WELL_KNOWN_CAPABILITIES.dns_internal).toBeDefined();
|
|
29
29
|
expect(WELL_KNOWN_CAPABILITIES.dns_internal.canonical_hostname).toBe('dns-int');
|
|
30
|
-
|
|
30
|
+
// ISS-0156: the resolver moved to a protected zone (dmz) so it can see
|
|
31
|
+
// protected-zone query sources for split-horizon views.
|
|
32
|
+
expect(WELL_KNOWN_CAPABILITIES.dns_internal.required_zone).toBe('dmz');
|
|
31
33
|
expect(WELL_KNOWN_CAPABILITIES.dns_internal.zone_enforced).toBe(true);
|
|
32
34
|
});
|
|
33
35
|
|
|
@@ -140,10 +142,11 @@ describe('Well-Known Capabilities Registry', () => {
|
|
|
140
142
|
expect(result.error).toContain("Capability 'auth' requires zone='secure'");
|
|
141
143
|
});
|
|
142
144
|
|
|
143
|
-
test('should validate correct zone for dns_internal', () => {
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
145
|
+
test('should validate correct zone for dns_internal (dmz — ISS-0156)', () => {
|
|
146
|
+
expect(validateZoneRequirement('dns_internal', 'dmz').valid).toBe(true);
|
|
147
|
+
// The old `internal` placement is now rejected — the resolver must be in a
|
|
148
|
+
// protected zone to see protected-zone query sources for split-horizon.
|
|
149
|
+
expect(validateZoneRequirement('dns_internal', 'internal').valid).toBe(false);
|
|
147
150
|
});
|
|
148
151
|
|
|
149
152
|
test('should allow dns_registrar in any zone (zone_enforced=false)', () => {
|
|
@@ -231,8 +234,10 @@ describe('Well-Known Capabilities Registry', () => {
|
|
|
231
234
|
expect(WELL_KNOWN_CAPABILITIES.database.required_zone).toBe('secure');
|
|
232
235
|
});
|
|
233
236
|
|
|
234
|
-
test('internal
|
|
235
|
-
|
|
237
|
+
test('the internal DNS resolver lives in the dmz (ISS-0156)', () => {
|
|
238
|
+
// Moved out of `internal`: the resolver must see protected-zone query
|
|
239
|
+
// sources for source-based split-horizon views (v2/INTERNAL_DNS_ZONE_VIEWS.md).
|
|
240
|
+
expect(WELL_KNOWN_CAPABILITIES.dns_internal.required_zone).toBe('dmz');
|
|
236
241
|
});
|
|
237
242
|
});
|
|
238
243
|
});
|
|
@@ -74,12 +74,20 @@ export const WELL_KNOWN_CAPABILITIES: Record<string, WellKnownCapability> = {
|
|
|
74
74
|
|
|
75
75
|
/**
|
|
76
76
|
* dns_internal - Internal DNS resolver
|
|
77
|
-
* Example: Technitium
|
|
78
|
-
* Security:
|
|
77
|
+
* Example: Technitium / knot-unbound for split-horizon DNS
|
|
78
|
+
* Security: a PROTECTED zone (dmz), NOT `internal` (ISS-0156,
|
|
79
|
+
* v2/INTERNAL_DNS_ZONE_VIEWS.md). The resolver must see each querying client's
|
|
80
|
+
* real source IP to serve source-based split-horizon views; fw-main NATs
|
|
81
|
+
* protected↔`internal`, so an `internal`-placed resolver sees every protected
|
|
82
|
+
* query as fw-main's address and can't tell the zones apart. Placed in `dmz`
|
|
83
|
+
* (a protected zone — protected↔protected is not NAT'd) it sees real
|
|
84
|
+
* protected-zone sources; `internal` devices reach it via a firewall
|
|
85
|
+
* DNS-ingress DNAT. The operator accepted the modest posture change (the
|
|
86
|
+
* `internal` zone is itself un-managed). zone_enforced stays true — just to dmz.
|
|
79
87
|
*/
|
|
80
88
|
dns_internal: {
|
|
81
89
|
canonical_hostname: 'dns-int',
|
|
82
|
-
required_zone: '
|
|
90
|
+
required_zone: 'dmz',
|
|
83
91
|
zone_enforced: true,
|
|
84
92
|
data_schema: {
|
|
85
93
|
server: {
|
|
@@ -1090,7 +1090,7 @@ export const COMMANDS: CommandDef[] = [
|
|
|
1090
1090
|
},
|
|
1091
1091
|
{
|
|
1092
1092
|
name: 'proxmox',
|
|
1093
|
-
description: 'Proxmox cluster introspection (nodes, capacity)',
|
|
1093
|
+
description: 'Proxmox cluster introspection (nodes, capacity, instance sizing)',
|
|
1094
1094
|
subcommands: [
|
|
1095
1095
|
{
|
|
1096
1096
|
name: 'node',
|
|
@@ -1103,6 +1103,70 @@ export const COMMANDS: CommandDef[] = [
|
|
|
1103
1103
|
},
|
|
1104
1104
|
],
|
|
1105
1105
|
},
|
|
1106
|
+
{
|
|
1107
|
+
name: 'vm',
|
|
1108
|
+
description: 'celilo-provisioned VM sizing',
|
|
1109
|
+
subcommands: [
|
|
1110
|
+
{
|
|
1111
|
+
name: 'list',
|
|
1112
|
+
description: 'List celilo VMs with desired vs actual size',
|
|
1113
|
+
args: [{ name: 'service-id', description: 'Proxmox service (optional if only one)' }],
|
|
1114
|
+
},
|
|
1115
|
+
{
|
|
1116
|
+
name: 'resize',
|
|
1117
|
+
description: 'Resize a celilo VM (canonical size + reconcile)',
|
|
1118
|
+
args: [{ name: 'name', description: 'Instance name (module/hostname)' }],
|
|
1119
|
+
flags: [
|
|
1120
|
+
{ name: 'memory', description: 'New RAM in MB', takesValue: true },
|
|
1121
|
+
{ name: 'cpu', description: 'New vCPU count', takesValue: true },
|
|
1122
|
+
{ name: 'force', description: 'Override the node capacity check', takesValue: false },
|
|
1123
|
+
{
|
|
1124
|
+
name: 'allow-reboot',
|
|
1125
|
+
description: 'Approve the stop/start a resize needs',
|
|
1126
|
+
takesValue: false,
|
|
1127
|
+
},
|
|
1128
|
+
{ name: 'skip-backup', description: 'Skip the pre-resize backup', takesValue: false },
|
|
1129
|
+
{
|
|
1130
|
+
name: 'yes',
|
|
1131
|
+
description: 'Auto-approve reboot + backup (headless)',
|
|
1132
|
+
takesValue: false,
|
|
1133
|
+
},
|
|
1134
|
+
],
|
|
1135
|
+
},
|
|
1136
|
+
],
|
|
1137
|
+
},
|
|
1138
|
+
{
|
|
1139
|
+
name: 'ct',
|
|
1140
|
+
description: 'celilo-provisioned container (LXC) sizing',
|
|
1141
|
+
subcommands: [
|
|
1142
|
+
{
|
|
1143
|
+
name: 'list',
|
|
1144
|
+
description: 'List celilo containers with desired vs actual size',
|
|
1145
|
+
args: [{ name: 'service-id', description: 'Proxmox service (optional if only one)' }],
|
|
1146
|
+
},
|
|
1147
|
+
{
|
|
1148
|
+
name: 'resize',
|
|
1149
|
+
description: 'Resize a celilo container (canonical size + reconcile)',
|
|
1150
|
+
args: [{ name: 'name', description: 'Instance name (module/hostname)' }],
|
|
1151
|
+
flags: [
|
|
1152
|
+
{ name: 'memory', description: 'New RAM in MB', takesValue: true },
|
|
1153
|
+
{ name: 'cpu', description: 'New vCPU count', takesValue: true },
|
|
1154
|
+
{ name: 'force', description: 'Override the node capacity check', takesValue: false },
|
|
1155
|
+
{
|
|
1156
|
+
name: 'allow-reboot',
|
|
1157
|
+
description: 'Approve the stop/start a resize needs',
|
|
1158
|
+
takesValue: false,
|
|
1159
|
+
},
|
|
1160
|
+
{ name: 'skip-backup', description: 'Skip the pre-resize backup', takesValue: false },
|
|
1161
|
+
{
|
|
1162
|
+
name: 'yes',
|
|
1163
|
+
description: 'Auto-approve reboot + backup (headless)',
|
|
1164
|
+
takesValue: false,
|
|
1165
|
+
},
|
|
1166
|
+
],
|
|
1167
|
+
},
|
|
1168
|
+
],
|
|
1169
|
+
},
|
|
1106
1170
|
],
|
|
1107
1171
|
},
|
|
1108
1172
|
{
|
|
@@ -1,11 +1,20 @@
|
|
|
1
1
|
import { describe, expect, test } from 'bun:test';
|
|
2
|
+
import type { ModuleManifest } from '../../manifest/schema';
|
|
2
3
|
import {
|
|
3
4
|
type PollCandidate,
|
|
5
|
+
needsPreUpgradeBackup,
|
|
4
6
|
pickAutoUpgrade,
|
|
5
7
|
pickUpgradePolicy,
|
|
6
8
|
selectPollTargets,
|
|
7
9
|
} from './module-upgrade';
|
|
8
10
|
|
|
11
|
+
/** Minimal manifest fixture; only `hooks` matters for the backup gate. */
|
|
12
|
+
function manifest(hooks?: ModuleManifest['hooks']): ModuleManifest {
|
|
13
|
+
return { id: 'forgejo', name: 'Forgejo', version: '0.3.0', hooks } as ModuleManifest;
|
|
14
|
+
}
|
|
15
|
+
const withBackupHook = manifest({ on_backup: { script: './scripts/backup.ts', timeout: 300000 } });
|
|
16
|
+
const noBackupHook = manifest({ on_install: { script: './scripts/setup.ts', timeout: 180000 } });
|
|
17
|
+
|
|
9
18
|
describe('pickUpgradePolicy (ISS-0138 — config override > manifest default > by-semver)', () => {
|
|
10
19
|
test('operator config wins over the manifest default', () => {
|
|
11
20
|
expect(pickUpgradePolicy('always-safe', 'always-fast')).toBe('always-safe');
|
|
@@ -80,3 +89,23 @@ describe('selectPollTargets (ISS-0139 — opted-in + a newer registry version)',
|
|
|
80
89
|
expect(targets.map((t) => t.moduleId)).toEqual(['a']);
|
|
81
90
|
});
|
|
82
91
|
});
|
|
92
|
+
|
|
93
|
+
describe('needsPreUpgradeBackup (ISS-0168 — gate on the TARGET version manifest)', () => {
|
|
94
|
+
// The regression: the upgrade that FIRST adds on_backup must still back up.
|
|
95
|
+
// The installed version lacked the hook; the target (passed here) adds it.
|
|
96
|
+
test('safe posture + target adds on_backup → backs up', () => {
|
|
97
|
+
expect(needsPreUpgradeBackup('safe', withBackupHook)).toBe(true);
|
|
98
|
+
});
|
|
99
|
+
|
|
100
|
+
test('safe posture + target has no on_backup → does not back up', () => {
|
|
101
|
+
expect(needsPreUpgradeBackup('safe', noBackupHook)).toBe(false);
|
|
102
|
+
});
|
|
103
|
+
|
|
104
|
+
test('fast posture (patch/revision) never backs up, even with on_backup', () => {
|
|
105
|
+
expect(needsPreUpgradeBackup('fast', withBackupHook)).toBe(false);
|
|
106
|
+
});
|
|
107
|
+
|
|
108
|
+
test('safe posture + no hooks block at all → does not back up', () => {
|
|
109
|
+
expect(needsPreUpgradeBackup('safe', manifest(undefined))).toBe(false);
|
|
110
|
+
});
|
|
111
|
+
});
|
|
@@ -21,7 +21,11 @@ import { modules } from '../../db/schema';
|
|
|
21
21
|
import type { ModuleManifest } from '../../manifest/schema';
|
|
22
22
|
import { RegistryClient } from '../../registry/client';
|
|
23
23
|
import { createModuleBackup } from '../../services/backup-create';
|
|
24
|
-
import {
|
|
24
|
+
import {
|
|
25
|
+
type DeployPosture,
|
|
26
|
+
type UpgradePolicy,
|
|
27
|
+
resolveDeployPosture,
|
|
28
|
+
} from '../../services/deploy-posture';
|
|
25
29
|
import { runModuleHealthCheck } from '../../services/health-runner';
|
|
26
30
|
import { getModuleConfigValue } from '../../services/module-config';
|
|
27
31
|
import { deployModule } from '../../services/module-deploy';
|
|
@@ -62,6 +66,23 @@ export function pickAutoUpgrade(
|
|
|
62
66
|
return fromManifest ?? false;
|
|
63
67
|
}
|
|
64
68
|
|
|
69
|
+
/**
|
|
70
|
+
* Pure (Rule 10): does this upgrade need a pre-deploy backup? Yes IFF the
|
|
71
|
+
* posture is "safe" (a minor/major bump) AND the TARGET version's manifest
|
|
72
|
+
* declares an on_backup hook.
|
|
73
|
+
*
|
|
74
|
+
* The `targetManifest` MUST be the version being upgraded TO (read back after
|
|
75
|
+
* the def is refreshed), never the installed one — gating on the installed
|
|
76
|
+
* manifest skips the backup on the very upgrade that introduces the hook
|
|
77
|
+
* (ISS-0168).
|
|
78
|
+
*/
|
|
79
|
+
export function needsPreUpgradeBackup(
|
|
80
|
+
posture: DeployPosture,
|
|
81
|
+
targetManifest: ModuleManifest,
|
|
82
|
+
): boolean {
|
|
83
|
+
return posture === 'safe' && Boolean(targetManifest.hooks?.on_backup);
|
|
84
|
+
}
|
|
85
|
+
|
|
65
86
|
export interface PollCandidate {
|
|
66
87
|
moduleId: string;
|
|
67
88
|
installed: string;
|
|
@@ -116,13 +137,34 @@ async function upgradeOneModule(
|
|
|
116
137
|
flags: Record<string, string | boolean>,
|
|
117
138
|
): Promise<CommandResult> {
|
|
118
139
|
const moduleId = mod.id;
|
|
119
|
-
const manifest = mod.manifestData as ModuleManifest;
|
|
120
140
|
|
|
121
|
-
//
|
|
141
|
+
// Update FIRST (refresh stored def). Every downstream decision — the
|
|
142
|
+
// backup gate and the posture policy — must consult the TARGET version's
|
|
143
|
+
// manifest, not the installed one. Gating on the pre-update manifest skipped
|
|
144
|
+
// the on_backup hook on the very upgrade that INTRODUCES it (e.g. a stateful
|
|
145
|
+
// module that first ships backup support doesn't protect its own next
|
|
146
|
+
// upgrade) — ISS-0168.
|
|
147
|
+
const updated = await fetchAndUpdate(client, moduleId, targetVersion, db, flags);
|
|
148
|
+
if (updated.status !== 'success') {
|
|
149
|
+
const why = updated.status === 'failed' ? updated.error : updated.reason;
|
|
150
|
+
return { success: false, error: `Update failed for ${moduleId}: ${why}` };
|
|
151
|
+
}
|
|
152
|
+
|
|
153
|
+
// Read the target def back: fetchAndUpdate persisted it but doesn't return
|
|
154
|
+
// the manifest. Fall back to the pre-update manifest only if the row somehow
|
|
155
|
+
// vanished (it won't on the success path).
|
|
156
|
+
const updatedRow = db.select().from(modules).where(eq(modules.id, moduleId)).get();
|
|
157
|
+
const targetManifest =
|
|
158
|
+
(updatedRow?.manifestData as ModuleManifest | undefined) ??
|
|
159
|
+
(mod.manifestData as ModuleManifest);
|
|
160
|
+
|
|
161
|
+
// Posture. Version delta is installed→target; the policy comes from the
|
|
162
|
+
// TARGET manifest (the version being installed declares its upgrade risk),
|
|
163
|
+
// with an operator config override still winning.
|
|
122
164
|
const configPolicy = getModuleConfigValue(moduleId, 'upgrade_policy');
|
|
123
165
|
const modulePolicy = pickUpgradePolicy(
|
|
124
166
|
typeof configPolicy?.value === 'string' ? configPolicy.value : undefined,
|
|
125
|
-
(
|
|
167
|
+
(targetManifest as ModuleManifest & { upgrade_policy?: string }).upgrade_policy,
|
|
126
168
|
);
|
|
127
169
|
// Per-release deploy_posture override lives in the .netapp release metadata;
|
|
128
170
|
// reading it requires fetching the package first. Deferred — the classifier
|
|
@@ -135,27 +177,18 @@ async function upgradeOneModule(
|
|
|
135
177
|
});
|
|
136
178
|
log.info(`Upgrading ${moduleId} ${mod.version} → ${targetVersion} (${posture} — ${reason})`);
|
|
137
179
|
|
|
138
|
-
//
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
if (posture === 'safe') {
|
|
147
|
-
if (manifest.hooks?.on_backup) {
|
|
148
|
-
const backup = await createModuleBackup(moduleId);
|
|
149
|
-
if (!backup.success) {
|
|
150
|
-
return {
|
|
151
|
-
success: false,
|
|
152
|
-
error: `Pre-upgrade backup failed for ${moduleId}: ${backup.error}`,
|
|
153
|
-
};
|
|
154
|
-
}
|
|
155
|
-
log.success(`Backed up ${moduleId} before deploy`);
|
|
156
|
-
} else {
|
|
157
|
-
log.warn(`${moduleId} has no on_backup hook — proceeding without a pre-upgrade backup`);
|
|
180
|
+
// Safe → back up first, gated on the TARGET manifest's on_backup hook.
|
|
181
|
+
if (needsPreUpgradeBackup(posture, targetManifest)) {
|
|
182
|
+
const backup = await createModuleBackup(moduleId);
|
|
183
|
+
if (!backup.success) {
|
|
184
|
+
return {
|
|
185
|
+
success: false,
|
|
186
|
+
error: `Pre-upgrade backup failed for ${moduleId}: ${backup.error}`,
|
|
187
|
+
};
|
|
158
188
|
}
|
|
189
|
+
log.success(`Backed up ${moduleId} before deploy`);
|
|
190
|
+
} else if (posture === 'safe') {
|
|
191
|
+
log.warn(`${moduleId} has no on_backup hook — proceeding without a pre-upgrade backup`);
|
|
159
192
|
}
|
|
160
193
|
|
|
161
194
|
// Deploy (idempotent).
|