@astrofoundry/pi-astro 0.35.0 → 0.36.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +3 -3
- package/agents/arcane.md +3 -1
- package/agents/release-reviewer.md +2 -2
- package/extensions/astro-discord/daily-report.test.ts +153 -77
- package/extensions/astro-discord/daily-report.ts +322 -53
- package/extensions/astro-discord/index.ts +180 -51
- package/extensions/astro-discord/reports.ts +0 -2
- package/extensions/astro-discord/tools-report.test.ts +32 -4
- package/extensions/astro-discord/tools-report.ts +51 -22
- package/extensions/specialist-gate/approvals.test.ts +3 -0
- package/package.json +1 -1
- package/skills/arcane/SKILL.md +16 -3
- package/specialists/AGENTS.md +3 -2
- package/specialists/README.md +5 -5
- package/specialists/arcane/run.ts +223 -10
- package/specialists/lib/stack-update.test.ts +101 -0
- package/specialists/lib/stack-update.ts +115 -0
- package/specialists/wrappers.test.ts +1 -0
package/skills/arcane/SKILL.md
CHANGED
|
@@ -38,9 +38,22 @@ The output is JSON with `counts`, `rows`, and `table`. Answer with the `table` f
|
|
|
38
38
|
- A note "major version change" or "newer major ... available" means the update crosses a major version. Read its release notes yourself before asking anything (next section).
|
|
39
39
|
- Private images (the registry refuses anonymous access, such as our own Grimoire builds) are left out of the table and the counts; the summary line counts them and `privateImages` lists them. Mention them only when the caller asks.
|
|
40
40
|
|
|
41
|
-
Messages that start with "Arcane Daily Report:" are the bridge's scheduled report, built from `update-report` every morning without you. When the caller replies to one, its list is the context ("update these", "why is this
|
|
41
|
+
Messages that start with "Arcane Daily Report:" are the bridge's scheduled report, built from `update-report` every morning without you: one section per stack with updates, a release-notes verdict on each major version, and an Update button per stack that owners press, which runs `stack-update` without you. When the caller replies to one, its list is the context ("update these", "why is this a new build"); run `update-report` again only when the caller wants fresh data.
|
|
42
42
|
|
|
43
|
-
|
|
43
|
+
## Applying updates
|
|
44
|
+
|
|
45
|
+
Apply updates only when asked, one stack at a time, with `["stack-update", "<stack>"]`. It applies every pending update of the stack in one call, without judgment:
|
|
46
|
+
|
|
47
|
+
- Pinned tags (`edit tag`) and digests (`edit digest`) move in the stack's Compose file through the GitOps checkout: commit, push, then a wait of up to 20 minutes for the stack's sync to report success at that commit. Only `image:` lines that name exactly the old reference change.
|
|
48
|
+
- `projects redeploy <stack> --yes` then pulls and restarts the stack (up to 35 minutes for large pulls).
|
|
49
|
+
- Checks: every service has a container that runs (a one-shot job may have exited 0), none is unhealthy or still starting after 5 minutes, and a fresh check finds every updated image current.
|
|
50
|
+
- A failed check after a Compose change reverts the commit, pushes, waits for the sync, and redeploys the previous file. A pull alone cannot be undone; say so when it fails.
|
|
51
|
+
|
|
52
|
+
The answer is JSON with `status`: `updated`, `rolled back`, `failed` (with `reason`), or `nothing to do`, plus `services` (`service`, `from`, `to`, `applied`: `pull` or the new reference, `result`: `current`, `behind`, `unchecked`) and `commit`/`revert` when the Compose file changed. Report it in plain words.
|
|
53
|
+
|
|
54
|
+
It refuses `arcane` (next section), stopped stacks (a redeploy would start them), a checkout with local or unpushed changes, and held stacks. Held stacks are listed in `update-holds.json` at the root of the GitOps repository, `{ "<stack>": "<reason>" }`; `update-report` returns them in `holds`, and the daily report shows the reason without a button. Change that file only when the caller asks, through the `git` commands.
|
|
55
|
+
|
|
56
|
+
An update across a major version is refused without `--confirm`: summarise its release notes first (next section), get the caller's go, then run `["stack-update", "<stack>", "--confirm"]`, which asks an owner for approval.
|
|
44
57
|
|
|
45
58
|
## Release notes
|
|
46
59
|
|
|
@@ -50,7 +63,7 @@ For every update that crosses a major version, and whenever the caller asks what
|
|
|
50
63
|
|
|
51
64
|
1. Run `release-notes` for that service and read every release it returns.
|
|
52
65
|
2. Summarise in a few lines: breaking changes, manual steps (database or data migrations, new, renamed, or removed settings and environment variables, changed volumes or ports), and what the Docker or upgrade sections require. Say plainly when the notes show none, and name the sections that were omitted.
|
|
53
|
-
3. Ask the caller for one go for that stack, with the summary. Do not apply a major update without it
|
|
66
|
+
3. Ask the caller for one go for that stack, with the summary. Do not apply a major update without it; with the go, run `stack-update <stack> --confirm`.
|
|
54
67
|
|
|
55
68
|
Minor and patch updates need no notes unless the caller asks. When `release-notes` reports no source, a source outside GitHub, or no releases, say so and ask for the go with that caveat; never guess the changes.
|
|
56
69
|
|
package/specialists/AGENTS.md
CHANGED
|
@@ -41,7 +41,7 @@ Wrappers ship as TypeScript and run under Node 24 type stripping: erasable synta
|
|
|
41
41
|
- Fixed remote commands: `DMZ_COMMANDS`, `PULSAR_COMMANDS`, `VPS_COMMANDS`, `OBS_COMMANDS`, `BACKUP_COMMANDS`, `HOST_COMMANDS`, and `INFERENCE_COMMANDS` must match the entry scripts in `entry/remote/` and their tracked copies in the homelab repository (`02-pulsar-proxmox/dmz/system/`, `02-pulsar-proxmox/pulsar/system/`, `00-frontdoor-vps/system/`, `02-pulsar-proxmox/observability/system/`, `04-nexus-macstudio/system/`, and `host/` of the `astronaute77/arcane` repository). Pulsar's root `authorized_keys` carries one forced key per specialist (network, backup, proxmox), each pinned to its own entry script. Change both sides in the same commit and redeploy the remote script. A new privileged step on the VPS also needs its exact line in `spc-edge-frontdoor-sudoers`.
|
|
42
42
|
- Repository access is `lib/repo.ts` (`repoCommand`, `RepoSpec`, `trackedFile`): one clone per specialist under `~/.specialists/work/<checkoutName>` (`homelab-edge`, `homelab-identity`; never share a checkout between specialists), a write deploy key, and an optional `writeRoot` that limits `git write`. `edge` (`deploy-nginx`) and `identity` (`dmz deploy-config`) deploy only what origin has: `trackedFile` refuses a dirty, ahead, or behind checkout. The DMZ entry keeps the previous template and restores it when render or restart fails.
|
|
43
43
|
- `network` firewall policy writes are `policyWrite`: JSON body from the agent, checked for the documented required fields, sent as `POST`, `PUT`, or `DELETE` through the same pinned request; UniFi validates the rest.
|
|
44
|
-
- Pi executes sibling tool calls in parallel, so two wrapper processes of the same specialist can run at once. `lib/lock.ts` (`withLock`, a directory lock under `~/.specialists/work/locks`, stale after 10 minutes) serialises shared resources; `edge` holds `edge-tunnel` around the fixed IAP port and the gcloud configuration directory. `repo.ts` checkouts are not locked yet.
|
|
44
|
+
- Pi executes sibling tool calls in parallel, so two wrapper processes of the same specialist can run at once. `lib/lock.ts` (`withLock`, a directory lock under `~/.specialists/work/locks`, stale after 10 minutes) serialises shared resources; `edge` holds `edge-tunnel` around the fixed IAP port and the gcloud configuration directory. `arcane` holds `arcane-gitops` (stale after 3 hours) around `git`, `manager-upgrade`, and `stack-update`, because an update keeps the checkout for up to about two hours; the other `repo.ts` checkouts are not locked yet.
|
|
45
45
|
- `edge` retries one SSH call through the IAP tunnel when the first connection is closed during setup (`isTransientSshFailure`: exit 255 with "closed by remote host" or a reset), except `reboot`. The IAP tunnel is `lib/tunnel.ts`: `gcloud compute start-iap-tunnel <instance> 22 --local-host-port=localhost:<port>` (a documented flag; the hidden `--listen-on-stdin` is not used), ready when the port accepts a connection, closed after the ssh call. `gcloud auth activate-service-account` runs before every tunnel with `CLOUDSDK_CONFIG` under `~/.specialists/work/gcloud` and `CLOUDSDK_PYTHON` from the config, because the Homebrew cask ships no interpreter.
|
|
46
46
|
- `dns` builds every Technitium call from `TECHNITIUM_READS` and `TECHNITIUM_WRITES`; `key=value` record parameters pass through by name (`token` and `node` refused) because the API documents dozens of type-specific parameters. Writes are refused on the secondary in code: the catalog zone is the only replication path.
|
|
47
47
|
- `security` parses Wazuh alert lines locally (`summariseWazuh`, `summariseVulns`) so the guest never needs `jq`; `wazuh-vulns` is a local view over the same `wazuh-alerts` dump, not a guest command.
|
|
@@ -55,7 +55,8 @@ Wrappers ship as TypeScript and run under Node 24 type stripping: erasable synta
|
|
|
55
55
|
- Every wrapper answers `--help` without reading config, so agents can discover subcommands before anything else is set up.
|
|
56
56
|
- `entry/specialist-install.sh` runs as root, so it runs only `/usr/bin`, `/bin`, `/usr/sbin`, `/sbin` and the Node it installed, never Homebrew or anything else the main user can write; it checks that `/usr/local/libexec/specialist-node` (and each `specialist-<tool>` folder) and its parents belong to root and are writable by nobody else. Arguments are checked before the root test (tests exercise that part). Downloads go to a root-only temporary directory and are verified before use: Node against `SHASUMS256.txt` and the LTS flag in `index.json`, npm tarballs against the registry's `sha512` integrity, arcane-cli against the release's checksums file. Inside `/Users/specialist` it works only as `specialist` (`sudo -u specialist -H`), from a staging directory root hands over, so a link the specialist plants cannot redirect a root write. Each component switches by rename after a check (every installed wrapper's `--help`, `--version` of the new binary) and restores the previous copy when the check fails. Older versions are refused; `rollback` swaps current and previous. `python`, `gcloud`, `bw`, and `jq` follow the same rules: version folders under `/usr/local/libexec/specialist-<tool>/` behind the links `current` and `previous`, sealed `root:wheel` and not writable by others, smoke-tested as `specialist` before `current` moves; `python` refuses a switch that breaks the current gcloud. Consumers use the `current` paths: `provision.sh` for `bw` and `jq`, `edge.json` for `gcloud` and `gcloudPython`. Nothing a wrapper runs with secrets may come from Homebrew.
|
|
57
57
|
- `main()` waits for stdout and stderr to flush before `process.exit`: on a pipe, exiting right after a write drops everything past the 64 KiB buffer, which cut large answers such as `projects list --all` until 2026-09-25 (`lib/flush.test.ts`).
|
|
58
|
-
- `arcane update-report [--all]` is `lib/image-updates.ts` over `lib/registry.ts`: a fresh `images updates check --all`, then `projects list --all` for the stack and service of each image reference, then the registries' tag lists and manifest digests (anonymous bearer tokens from each registry's challenge; digests by HEAD, which Docker Hub does not count as a pull; lscr.io is read from ghcr.io). A floating or major-series tag is resolved to the release tag whose digest matches the running and the latest build, searching the newest tags of the three commonest release shapes; a fixed tag is compared with the newest tag of the same shape; a digest pin with the newest release. Checks older than one minute before the run count as stale and show as unknown. Private images (anonymous access refused with 401 or 403) are excluded from rows, counts, and table, listed in `privateImages`, and counted in the summary line. The table is fixed-width text for a code block, because Discord renders no Markdown tables.
|
|
58
|
+
- `arcane update-report [--all]` is `lib/image-updates.ts` over `lib/registry.ts`: a fresh `images updates check --all`, then `projects list --all` for the stack and service of each image reference, then the registries' tag lists and manifest digests (anonymous bearer tokens from each registry's challenge; digests by HEAD, which Docker Hub does not count as a pull; lscr.io is read from ghcr.io). A floating or major-series tag is resolved to the release tag whose digest matches the running and the latest build, searching the newest tags of the three commonest release shapes; a fixed tag is compared with the newest tag of the same shape; a digest pin with the newest release. Checks older than one minute before the run count as stale and show as unknown. Private images (anonymous access refused with 401 or 403) are excluded from rows, counts, and table, listed in `privateImages`, and counted in the summary line. The table is fixed-width text for a code block, because Discord renders no Markdown tables. The answer adds `holds` from `update-holds.json` at origin's branch head (`git show`, no working-tree read), or `holdsError` when that file is malformed.
|
|
59
|
+
- `arcane stack-update <stack> [--confirm]` (`lib/stack-update.ts` for the pure parts) is deterministic: refuses `arcane`, held stacks (the checkout's `update-holds.json` after `pull --ff-only`; a malformed file is an error), a dirty or ahead checkout, stopped stacks, and an update across a major version without `--confirm` (which the gate turns into an owner approval). Pending rows come from a fresh per-image check resolved like `update-report`. `edit tag`/`edit digest` rows rewrite `image:` lines that name exactly the old reference (`replaceImageRef`, quoted or not, trailing comment kept) in the sync's `composePath`, then commit, push, and wait for `gitops list` to report success at that commit (20 min, polled every 30 s). Then `projects redeploy <stack> --yes` (35 min), container checks through `containers list --all` and `containers get` (`composeInfo.projectName`, health; 5 min, polled every 15 s), and a fresh check that each service is current. A failure after a commit reverts, pushes, waits for the sync, redeploys, and checks again (`rolled back`); a pull-only failure stays `failed`. Every outcome prints JSON with `status` and exits 0. Worst case about 125 minutes, so the gate's `timeoutMs` for arcane must exceed it (8,100,000). The checkout is not locked, so an agent call and a bridge click at the same time can collide on it; the bridge runs one Arcane update at a time.
|
|
59
60
|
- `arcane host <command>` runs over the forced-command key of user `spc-arcane` on VM 100. `HOST_COMMANDS` in `lib/arcane-host.ts` must list exactly the operations of `entry/remote/spc-arcane-op.py` (a test compares them); the tracked copies live in the arcane repository's `host/` (`spc-arcane-host-entry`, `spc-arcane-op`, `spc-arcane-sudoers`, `spc-arcane-install.sh`) with the probes in `host/spc-arcane-probes/`. The entry only splits words (letters, digits, space, `._:/+-`) and hands them to `sudo -n /usr/local/sbin/spc-arcane-op`, the one command its sudoers line allows; the op checks the name and every argument and runs the reviewed arcane-ops probes, which print only allowlisted fields. Manager steps (`manager-*`) are internal to `manager-upgrade`/`manager-rollback` and refused through `host`. Nothing on the host deletes data: a manager rollback restores the image and Compose file, not the data volume (R3 stays manual), `db-restore` only fills an empty database, and dumps are never pruned by the tool. `host-audit` compares the host's hashes with `host/<name>` of the `arcane-gitops` checkout after a pull.
|
|
60
61
|
- `arcane manager-upgrade <version>` (`lib/arcane-manager.ts`) follows arcane/AGENTS.md "Manager upgrades": the official image's revision label must equal the release tag's commit (GitHub API), `manager-prepare` records `compose.before.yaml` and every container before the pin commit, the pin goes through the `arcane-gitops` checkout (`git` is exported from `lib/repo.ts` for this), the wait polls `gitops list` and the deployed Compose hash every 30 s for up to 30 minutes, then pull, cold backup, `up -d --no-deps --pull never`, and checks. Any failure before the restart reverts the pin (`stopped`); a failed check after it reverts the pin and runs `manager-rollback` (`rolled back`). The answer always exits 0 with `status`, so the bridge and the agent both see it.
|
|
61
62
|
- `media` runs trendarr unchanged: `trendarrCli` in `media.json` is its `dist/cli.js`, started with `process.execPath` because sudo's PATH has no node. Each call gets `HOME` set to a fresh `0700` directory under `~/.specialists/work/media/` that holds only `.trendarr/config.json` (`0600`, trendarr's own schema, built from the note fields and `media.json`) and is removed in `finally`; trendarr reads no environment variables and has no config flag, and Node's `os.homedir()` follows `$HOME` on POSIX. Every call passes `--json`; `init` is refused. An add prints trendarr's per-ID `results` and exits 0 even when trendarr exits 1 for a failed ID, because the gate shows only stderr on failure; other failures carry trendarr's stderr without its `warning:` lines, which go into the JSON as `warnings`. The bridge's slash commands and Add buttons call the same wrapper through the entry point (`extensions/astro-discord/media.ts`).
|
package/specialists/README.md
CHANGED
|
@@ -4,7 +4,7 @@ A specialist is a Pi subagent that is the only way to operate one area of the ho
|
|
|
4
4
|
|
|
5
5
|
| Agent | Area | What it can do |
|
|
6
6
|
|---|---|---|
|
|
7
|
-
| `astro.arcane` | Arcane container platform (VM 100) and its GitOps repository | Every `arcane-cli` operation except its own `config`, `auth`, `self-update`; `update-report` (fresh check of every stack with running and latest release versions); `release-notes <stack>/<service>` (GitHub notes between the two versions); `project-status`, `deployment check|wait`; `manager-upgrade <version>` and `manager-rollback <stamp>` (the documented manager upgrade, owner approval); `host <command>`: sanitized host checks (memory, stats, logs, OOM, kernel OOM, audit, capacity, storage, Docker usage, service and queue checks) and `db-list|db-dump|db-restore` for Mealie and Paperless through user `spc-arcane` on VM 100; Compose changes of GitOps-managed projects through `git show|write|commit|push`, then `arcane-cli gitops` sync |
|
|
7
|
+
| `astro.arcane` | Arcane container platform (VM 100) and its GitOps repository | Every `arcane-cli` operation except its own `config`, `auth`, `self-update`; `update-report` (fresh check of every stack with running and latest release versions); `release-notes <stack>/<service>` (GitHub notes between the two versions); `project-status`, `deployment check|wait`; `stack-update <stack>` (a stack's pending image updates through GitOps, redeploy, checks, and rollback; across a major version only with `--confirm`, owner approval; held stacks in the repository's `update-holds.json` are refused); `manager-upgrade <version>` and `manager-rollback <stamp>` (the documented manager upgrade, owner approval); `host <command>`: sanitized host checks (memory, stats, logs, OOM, kernel OOM, audit, capacity, storage, Docker usage, service and queue checks) and `db-list|db-dump|db-restore` for Mealie and Paperless through user `spc-arcane` on VM 100; Compose changes of GitOps-managed projects through `git show|write|commit|push`, then `arcane-cli gitops` sync |
|
|
8
8
|
| `astro.identity` | Zitadel; Pomerium, nginx, lego on VM 104; `02-pulsar-proxmox/dmz/` in the homelab repository | Any Zitadel API call (v2, management, admin, auth) with responses redacted; `dmz status|journal|config-template|render|pomerium-restart|nginx-reload|lego-renew|deploy-config`; `git` on the homelab checkout, writes limited to `02-pulsar-proxmox/dmz/` |
|
|
9
9
|
| `astro.network` | UniFi, FreeRADIUS (LXC 108), Tailscale (LXC 105) | Read UniFi sites, devices, clients, networks, zones, firewall policies, WANs; create, update, delete firewall policies; RADIUS and Tailscale checks on Pulsar |
|
|
10
10
|
| `astro.dns` | Technitium primary (Synapse) and secondary (LXC 107); Cloudflare zones `37pla.net`, `monadeo.com` | Read zones and records on both resolvers, resolve, stats; create and delete zones, add, update, delete records on the primary; Cloudflare zones and record add, update, delete |
|
|
@@ -122,7 +122,7 @@ Host: Cortex (macOS on Apple silicon; Homebrew git; the specialists' Node, trend
|
|
|
122
122
|
```json
|
|
123
123
|
{ "entry": "/usr/local/bin/specialist-cli", "user": "specialist",
|
|
124
124
|
"specialists": [
|
|
125
|
-
{ "agent": "astro.arcane", "tool": "arcane", "service": "arcane", "timeoutMs":
|
|
125
|
+
{ "agent": "astro.arcane", "tool": "arcane", "service": "arcane", "timeoutMs": 8100000 },
|
|
126
126
|
{ "agent": "astro.identity", "tool": "identity", "service": "identity", "timeoutMs": 120000 },
|
|
127
127
|
{ "agent": "astro.network", "tool": "network", "service": "network", "timeoutMs": 120000 },
|
|
128
128
|
{ "agent": "astro.dns", "tool": "dns", "service": "dns", "timeoutMs": 60000 },
|
|
@@ -140,11 +140,11 @@ Check: `sudo -n -u specialist -H /usr/local/bin/specialist-cli arcane --caller t
|
|
|
140
140
|
|
|
141
141
|
The `astro-discord` extension gives Discord users access to the specialists from a channel. It runs in a headless Pi host on Cortex (LaunchAgent `com.astrofoundry.astro-discord`, user `cortex`), never in interactive sessions.
|
|
142
142
|
|
|
143
|
-
- Channels are configured one by one (`channels.<id>`): `trigger` is `always` (every message that names a specialist counts) or `mention` (only messages that start with `@Cortex` or reply to the bot); `specialists` is `*` or a list. A channel with one specialist needs no prefix: every message there is a task for it. In a channel with several, `dns list the zones` names the specialist; an addressed message without a name gets a reply asking for one. Threads inherit their parent channel's entry; unlisted channels are ignored. The bot replies at once with "<specialist> is working on your task" and shows the typing indicator until the answer replaces that reply. Reactions on your message: ⏳ running, 🔒 waiting for an owner's approval, ✅ done, ❌ failed. `help` shows the specialists you may use in that channel and the commands (`status` lists running tasks; owners in the admin channel also see the `config` commands). Each channel and specialist has one conversation: a reply to the bot's answer, or another task for that specialist within 30 minutes, continues it with everything the specialist saw before; otherwise a new one starts, and a task beginning with `new` forces that. Replied-to and forwarded messages (text, embeds, attachment names) travel with the task, so "check this" on a forwarded alert works. Image attachments on the message or on what it quotes (up to 4, 8 MiB each) are downloaded and passed to the specialist, which sees them; other file types are named only. Sessions live under `~/.config/astro-discord/sessions/<specialist>/` and are deleted after 7 days. Long answers arrive as a `.md` attachment. When the bridge restarts, open approval requests are marked cancelled and running tasks interrupted in Discord; ask again. Scheduled reports are a list in the config, one entry per report: `"reports": [{ "report": "arcane", "channelId": "<id>", "time": "06:00", "timeZone": "Europe/Madrid" }, { "report": "tools", "channelId": "<admin channel id>", "time": "06:00", "timeZone": "Europe/Madrid" }]` (the earlier single `dailyReport` key is refused; move it into the list). The Arcane report posts an "Arcane Daily Report:" message every morning
|
|
143
|
+
- Channels are configured one by one (`channels.<id>`): `trigger` is `always` (every message that names a specialist counts) or `mention` (only messages that start with `@Cortex` or reply to the bot); `specialists` is `*` or a list. A channel with one specialist needs no prefix: every message there is a task for it. In a channel with several, `dns list the zones` names the specialist; an addressed message without a name gets a reply asking for one. Threads inherit their parent channel's entry; unlisted channels are ignored. The bot replies at once with "<specialist> is working on your task" and shows the typing indicator until the answer replaces that reply. Reactions on your message: ⏳ running, 🔒 waiting for an owner's approval, ✅ done, ❌ failed. `help` shows the specialists you may use in that channel and the commands (`status` lists running tasks; owners in the admin channel also see the `config` commands). Each channel and specialist has one conversation: a reply to the bot's answer, or another task for that specialist within 30 minutes, continues it with everything the specialist saw before; otherwise a new one starts, and a task beginning with `new` forces that. Replied-to and forwarded messages (text, embeds, attachment names) travel with the task, so "check this" on a forwarded alert works. Image attachments on the message or on what it quotes (up to 4, 8 MiB each) are downloaded and passed to the specialist, which sees them; other file types are named only. Sessions live under `~/.config/astro-discord/sessions/<specialist>/` and are deleted after 7 days. Long answers arrive as a `.md` attachment. When the bridge restarts, open approval requests are marked cancelled and running tasks interrupted in Discord; ask again. Scheduled reports are a list in the config, one entry per report: `"reports": [{ "report": "arcane", "channelId": "<id>", "time": "06:00", "timeZone": "Europe/Madrid" }, { "report": "tools", "channelId": "<admin channel id>", "time": "06:00", "timeZone": "Europe/Madrid" }]` (the earlier single `dailyReport` key is refused; move it into the list). The Arcane report posts an "Arcane Daily Report:" message every morning from a fresh `update-report`: one section per stack with updates, each service as current to target version with the kind of change, a release-notes verdict on every major version, and an Update button per stack that only owners can press (it runs `arcane stack-update <stack>`; the button turns into the outcome and the reason arrives as a note only the clicker sees). Stacks held in `update-holds.json` at the root of the arcane GitOps repository (`{ "supabase": "coordinated release: review first" }`) show the reason and get no button. Replying to the report asks the arcane specialist with the report as context. The tools report posts "Cortex Tools Report:": every tool on Cortex with a newer version (Homebrew packages installed on request and casks, Pi, pi-astro, and through `specialist-install` the wrappers, arcane-cli at the Arcane server's version, trendarr, and the specialist Node at the latest LTS), a release-notes verdict from `astro.release-reviewer` for the tools the specialists build on, an Update button per line, and one Upgrade everything for Homebrew. Only owners can press them; nothing is installed otherwise. After an update that needs it, the bridge, herdr, or Astrogate restarts as soon as nothing runs on it; after a day of waiting, a Restart now button appears. The bridge calls `sudo -n /usr/local/sbin/specialist-install` through the sudoers rule from step 4.
|
|
144
144
|
- Media: where the `media` specialist is present (`config channel <#channel> always media`), `/trending [type] [min_rating] [count]`, `/search <title> [type]`, and `/downloads <status|pause|resume|unpause>` run the wrapper without a model. Lists arrive as two messages, Movies and Series, each title with a button beside it: **Add** adds it at once and turns into Added, In library, or Not found (Retry after a failure); a greyed-out In library or No IMDb ID button needs nothing. Notes such as "downloads are paused" go only to the user who clicked. Owners and `access.media` users may use them; anyone else gets a refusal only they see, and in a channel without `media` the bot names the channels that have it. Plain messages in the channel still go to the agent, for requests by name ("add the new Dune"). At startup the bridge sets the three commands as the application's global commands (servers only).
|
|
145
|
-
- Arcane manager: when the morning Arcane Daily Report shows a newer manager release, the
|
|
145
|
+
- Arcane manager: when the morning Arcane Daily Report shows a newer manager release, the `arcane` section carries an **Upgrade Arcane to vX** button. Only an owner's click on the newest report runs `arcane manager-upgrade <version>` through the entry point (one at a time, counted as open work so no bridge restart interrupts it); the button turns into the outcome, a reply gives the result, and a successful upgrade installs the matching `arcane-cli` with `specialist-install`.
|
|
146
146
|
- Access: `owners` may use every specialist and decide approvals; `access.<specialist>` lists extra users for that specialist; everyone else gets no reply. Owners change the configuration from the admin channel (`adminChannelId`) without a shell: `config show`, `config channel <#channel> <always|mention> <*|dns,edge>`, `config channel <#channel> remove`, `config access <specialist> add|remove <@user>`; the bot writes `~/.config/astro-discord/config.json` on Cortex, which is also editable by hand and re-read on every message. Deny the bot's role View Channel on channels it must never read; that is the boundary the map cannot provide.
|
|
147
|
-
- Approvals: a risky call (anything with `--confirm`, plus writes such as record or zone deletes, Zitadel changes, Pomerium deploys, service restarts, and on Arcane destructive actions: `down`, deletes, prunes, image, volume, or network removal, `system upgrade`, `gitops delete`, plus `manager-upgrade`, `manager-rollback`, `host db-restore`, and the Radarr host-config reads; help lookups never ask) pauses the specialist and posts Approve/Deny buttons; only owners' clicks count; no decision within `approvalTimeoutMinutes` denies it. The specialist then reports the denial. Ambiguous tasks get a question back instead of an action.
|
|
147
|
+
- Approvals: a risky call (anything with `--confirm`, plus writes such as record or zone deletes, Zitadel changes, Pomerium deploys, service restarts, and on Arcane destructive actions: `down`, deletes, prunes, image, volume, or network removal, `system upgrade`, `gitops delete`, plus `manager-upgrade`, `manager-rollback`, `host db-restore`, and the Radarr host-config reads; `stack-update` asks only across a major version, through its `--confirm`; an owner's Update click on the report needs no approval; help lookups never ask) pauses the specialist and posts Approve/Deny buttons; only owners' clicks count; no decision within `approvalTimeoutMinutes` denies it. The specialist then reports the denial. Ambiguous tasks get a question back instead of an action.
|
|
148
148
|
- Setup, as `cortex`: `d=$(mktemp -d) && cp ~/.pi/agent/npm/node_modules/@astrofoundry/pi-astro/extensions/astro-discord/*.ts "$d" && node --disable-warning=ExperimentalWarning "$d/setup.ts"` (Node does not strip types inside `node_modules`, so the files are copied first) prompts for the bot token and the ids, checks each against Discord, and writes `~/.config/astro-discord/{token,config.json,run.sh}` and the LaunchAgent; it prints the `launchctl bootstrap` line. After a pi-astro update restart the bridge with `launchctl kickstart -k gui/$(id -u)/com.astrofoundry.astro-discord`. Log: `~/Library/Logs/astro-discord.log`.
|
|
149
149
|
- Discord side: application `Cortex` with a bot, Message Content Intent enabled, invited with scopes `bot` and `applications.commands` and permissions View Channel, Send Messages, Attach Files, Read Message History, Add Reactions.
|
|
150
150
|
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { createHash } from "node:crypto";
|
|
2
|
-
import { mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs";
|
|
2
|
+
import { existsSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs";
|
|
3
3
|
import { tmpdir } from "node:os";
|
|
4
4
|
import { dirname, join } from "node:path";
|
|
5
5
|
import { buildHostRemote, compareAudit, HOST_COMMANDS, type AuditFile } from "../lib/arcane-host.ts";
|
|
@@ -7,14 +7,16 @@ import { failedChecks, MANAGER_COMPOSE, MANAGER_IMAGE, normalizeVersion, olderTh
|
|
|
7
7
|
import { readConfig } from "../lib/config.ts";
|
|
8
8
|
import { containerFacts, deploymentChecks, listRows, parseDeploymentArgs, syncFacts, type DeploymentExpectation } from "../lib/deployment.ts";
|
|
9
9
|
import { ServiceError, UsageError } from "../lib/errors.ts";
|
|
10
|
+
import { withLock } from "../lib/lock.ts";
|
|
10
11
|
import { main } from "../lib/main.ts";
|
|
11
|
-
import { buildUpdateReport, parseTag, resolveRef, stackServices, type CheckEntry } from "../lib/image-updates.ts";
|
|
12
|
+
import { buildUpdateReport, parseTag, resolveRef, stackServices, type CheckEntry, type UpdateRow } from "../lib/image-updates.ts";
|
|
12
13
|
import { printJson, printRaw } from "../lib/output.ts";
|
|
13
14
|
import { specialistsHome } from "../lib/paths.ts";
|
|
14
15
|
import { run, type RunResult } from "../lib/proc.ts";
|
|
15
16
|
import { HttpRegistry, parseImageRef } from "../lib/registry.ts";
|
|
16
17
|
import { releaseNotesFor } from "../lib/releases.ts";
|
|
17
|
-
import { ensureCheckout, git, REPO_HELP, repoCommand, repoStatus, type RepoSpec } from "../lib/repo.ts";
|
|
18
|
+
import { ensureCheckout, git, REPO_HELP, repoCommand, repoStatus, safeRepoPath, type RepoSpec } from "../lib/repo.ts";
|
|
19
|
+
import { commitMessage, containersProblem, crossesMajor, HOLDS_FILE, nextImageRef, parseHolds, pendingRows, replaceImageRef, type ServiceResult, type StackContainer, type StackStatus } from "../lib/stack-update.ts";
|
|
18
20
|
import { ARCANE_DENIED_KEYS, filterJsonText, stripKeys } from "../lib/sanitize.ts";
|
|
19
21
|
import { readSecret, secretPath } from "../lib/secrets.ts";
|
|
20
22
|
import { sshFixed } from "../lib/ssh.ts";
|
|
@@ -99,8 +101,15 @@ ${Object.entries(HOST_COMMANDS)
|
|
|
99
101
|
.map(([, c]) => ` ${c.help}`)
|
|
100
102
|
.join("\n")}
|
|
101
103
|
update-report [--all] fresh image update check for every stack, with running and latest release versions
|
|
102
|
-
from the registries; JSON with a ready table (--all also lists up-to-date services)
|
|
103
|
-
Checks only: never pulls images or redeploys.
|
|
104
|
+
from the registries; JSON with a ready table (--all also lists up-to-date services) and
|
|
105
|
+
the held stacks of ${HOLDS_FILE}. Checks only: never pulls images or redeploys.
|
|
106
|
+
stack-update <stack> [--confirm]
|
|
107
|
+
apply every pending update of one stack: pinned tags and digests move in the stack's
|
|
108
|
+
Compose file (commit, push, wait for the sync), then the stack is redeployed with a pull
|
|
109
|
+
and checked (containers running and healthy, images current). A failed check after a
|
|
110
|
+
Compose change reverts it and redeploys; a pull alone cannot be undone. Refuses held
|
|
111
|
+
stacks, stopped stacks, and arcane (manager-upgrade). An update across a major version
|
|
112
|
+
needs --confirm, which asks an owner for approval.
|
|
104
113
|
release-notes <stack>[/<service>]
|
|
105
114
|
GitHub release notes between the running and the latest version of one service's image,
|
|
106
115
|
found through the image's org.opencontainers.image.source label
|
|
@@ -160,14 +169,14 @@ function writeCallConfig(config: ArcaneConfig, token: string): string {
|
|
|
160
169
|
return file;
|
|
161
170
|
}
|
|
162
171
|
|
|
163
|
-
type Cli = (args: string[]) => Promise<{ stdout: string; stderr: string; code: number }>;
|
|
172
|
+
type Cli = (args: string[], timeoutMs?: number) => Promise<{ stdout: string; stderr: string; code: number }>;
|
|
164
173
|
|
|
165
174
|
/** One federated token and one private config copy for every arcane-cli call of this invocation. */
|
|
166
175
|
async function withCli<T>(config: ArcaneConfig, fn: (cli: Cli) => Promise<T>): Promise<T> {
|
|
167
176
|
const token = await federatedToken(config);
|
|
168
177
|
const callConfig = writeCallConfig(config, token);
|
|
169
178
|
try {
|
|
170
|
-
return await fn((args) => run(config.arcaneCli, ["--config", callConfig, "--no-color", ...args], { env: { ARCANE_TOKEN: token }, timeoutMs
|
|
179
|
+
return await fn((args, timeoutMs = TIMEOUT_MS) => run(config.arcaneCli, ["--config", callConfig, "--no-color", ...args], { env: { ARCANE_TOKEN: token }, timeoutMs }));
|
|
171
180
|
} finally {
|
|
172
181
|
rmSync(dirname(callConfig), { recursive: true, force: true });
|
|
173
182
|
}
|
|
@@ -202,14 +211,30 @@ async function updateReport(config: ArcaneConfig, args: string[]): Promise<numbe
|
|
|
202
211
|
const extra = args.filter((a) => a !== "--all");
|
|
203
212
|
if (extra.length > 0) throw new UsageError(`update-report takes only --all, got: ${extra.join(" ")}`);
|
|
204
213
|
const startedAt = Date.now();
|
|
214
|
+
const holds = await holdsFromOrigin(config);
|
|
205
215
|
return withCli(config, async (cli) => {
|
|
206
216
|
const check = await cliJson(cli, ["images", "updates", "check", "--all"]);
|
|
207
217
|
const projects = await cliJson(cli, ["projects", "list", "--all"]);
|
|
208
|
-
|
|
218
|
+
const report = await buildUpdateReport({ check, projects, registry: new HttpRegistry(), startedAt, all: args.includes("--all") });
|
|
219
|
+
printJson({ ...report, holds: holds.holds, ...(holds.error ? { holdsError: holds.error } : {}) });
|
|
209
220
|
return 0;
|
|
210
221
|
});
|
|
211
222
|
}
|
|
212
223
|
|
|
224
|
+
/** The held stacks as pushed to the GitOps repository, read without touching the checkout's working tree. */
|
|
225
|
+
async function holdsFromOrigin(config: ArcaneConfig): Promise<{ holds: Record<string, string>; error?: string }> {
|
|
226
|
+
try {
|
|
227
|
+
const spec = repoSpec(config);
|
|
228
|
+
const dir = await ensureCheckout(spec);
|
|
229
|
+
await git(spec, dir, ["fetch", "--quiet", "origin", config.gitopsBranch]);
|
|
230
|
+
const listed = (await git(spec, dir, ["ls-tree", "--name-only", `origin/${config.gitopsBranch}`, HOLDS_FILE])).trim();
|
|
231
|
+
if (listed !== HOLDS_FILE) return { holds: {} };
|
|
232
|
+
return { holds: parseHolds(await git(spec, dir, ["show", `origin/${config.gitopsBranch}:${HOLDS_FILE}`])) };
|
|
233
|
+
} catch (err) {
|
|
234
|
+
return { holds: {}, error: `${HOLDS_FILE} unreadable: ${err instanceof Error ? err.message.slice(0, 200) : String(err)}` };
|
|
235
|
+
}
|
|
236
|
+
}
|
|
237
|
+
|
|
213
238
|
async function releaseNotes(config: ArcaneConfig, args: string[]): Promise<number> {
|
|
214
239
|
if (args.length !== 1 || !/^[A-Za-z0-9_.-]+(\/[A-Za-z0-9_.-]+)?$/.test(args[0])) throw new UsageError("release-notes <stack>[/<service>]");
|
|
215
240
|
const [stack, service] = args[0].split("/");
|
|
@@ -370,6 +395,16 @@ const SYNC_WAIT_MS = 30 * MINUTE;
|
|
|
370
395
|
* stopped (nothing was restarted). The exit code is 0 in all three so the
|
|
371
396
|
* caller always gets the answer; usage and precondition errors still fail.
|
|
372
397
|
*/
|
|
398
|
+
/**
|
|
399
|
+
* Everything that writes the arcane-gitops checkout takes turns: an update holds it
|
|
400
|
+
* for up to about two hours (sync and redeploy waits), so the lock outlives that.
|
|
401
|
+
*/
|
|
402
|
+
const CHECKOUT_STALE_MS = 3 * 60 * 60_000;
|
|
403
|
+
|
|
404
|
+
function checkoutLocked<T>(fn: () => Promise<T>): Promise<T> {
|
|
405
|
+
return withLock("arcane-gitops", fn, { staleMs: CHECKOUT_STALE_MS });
|
|
406
|
+
}
|
|
407
|
+
|
|
373
408
|
async function managerUpgrade(config: ArcaneConfig, args: string[]): Promise<number> {
|
|
374
409
|
if (args.length !== 1) throw new UsageError("manager-upgrade <version>");
|
|
375
410
|
const version = normalizeVersion(args[0]);
|
|
@@ -480,20 +515,198 @@ async function managerRollback(args: string[]): Promise<number> {
|
|
|
480
515
|
return 0;
|
|
481
516
|
}
|
|
482
517
|
|
|
518
|
+
// ---------------------------------------------------------------- stack updates
|
|
519
|
+
|
|
520
|
+
/** How long a stack's sync may take to report success at a new commit; syncs run every few minutes. */
|
|
521
|
+
const STACK_SYNC_WAIT_MS = 20 * MINUTE;
|
|
522
|
+
/** arcane-cli allows a project redeploy up to 30 minutes for large pulls. */
|
|
523
|
+
const REDEPLOY_TIMEOUT_MS = 35 * MINUTE;
|
|
524
|
+
/** Containers with a health check get this long to leave "starting". */
|
|
525
|
+
const HEALTH_WAIT_MS = 5 * MINUTE;
|
|
526
|
+
|
|
527
|
+
interface StackState {
|
|
528
|
+
rows: UpdateRow[];
|
|
529
|
+
services: number;
|
|
530
|
+
stopped: boolean;
|
|
531
|
+
composePath: string | null;
|
|
532
|
+
}
|
|
533
|
+
|
|
534
|
+
/** The stack's services with a fresh check of each of its images, resolved the way update-report resolves them. */
|
|
535
|
+
async function stackState(cli: Cli, stack: string, startedAt: number): Promise<StackState> {
|
|
536
|
+
const projects = await cliJson(cli, ["projects", "list", "--all"]);
|
|
537
|
+
const services = stackServices(projects).filter((s) => s.stack === stack);
|
|
538
|
+
if (services.length === 0) throw new UsageError(`no stack named ${stack}`);
|
|
539
|
+
const project = listRows(projects).find((p) => p.name === stack);
|
|
540
|
+
const sync = listRows(await cliJson(cli, ["gitops", "list", "--all"])).filter((s) => project !== undefined && s.projectId === project.id);
|
|
541
|
+
const composePath = sync.length === 1 && typeof sync[0].composePath === "string" ? sync[0].composePath : null;
|
|
542
|
+
const registry = new HttpRegistry();
|
|
543
|
+
const resolved = new Map<string, Awaited<ReturnType<typeof resolveRef>>>();
|
|
544
|
+
for (const ref of [...new Set(services.map((s) => s.ref).filter((r): r is string => r !== null))]) {
|
|
545
|
+
const entry = parseImageRef(ref).digest ? undefined : ((await cliJson(cli, ["images", "updates", "check", ref])) as CheckEntry);
|
|
546
|
+
resolved.set(ref, await resolveRef(ref, entry, registry, startedAt));
|
|
547
|
+
}
|
|
548
|
+
const rows: UpdateRow[] = services.map((s) => {
|
|
549
|
+
const r = s.ref === null ? undefined : resolved.get(s.ref);
|
|
550
|
+
if (s.ref === null || !r) return { stack, service: s.service, ref: s.ref ?? "", image: "?", kind: "floating", running: "?", latest: "?", status: "unknown", action: "", note: "Arcane lists no image for this service" };
|
|
551
|
+
return { stack, service: s.service, ref: s.ref, image: s.ref, kind: r.kind, running: r.running, latest: r.latest, status: r.status, action: r.action, note: r.note, ...(r.latestDigest ? { latestDigest: r.latestDigest } : {}) };
|
|
552
|
+
});
|
|
553
|
+
return { rows, services: services.length, stopped: services.some((s) => s.stopped), composePath };
|
|
554
|
+
}
|
|
555
|
+
|
|
556
|
+
/** The stack's containers as `containers get` reports them, found by the stack's images and kept when Compose says they belong to it. */
|
|
557
|
+
async function stackContainers(cli: Cli, stack: string, refs: readonly string[]): Promise<StackContainer[]> {
|
|
558
|
+
const listed = listRows(await cliJson(cli, ["containers", "list", "--all"])).filter((c) => typeof c.image === "string" && refs.includes(c.image));
|
|
559
|
+
const out: StackContainer[] = [];
|
|
560
|
+
for (const c of listed) {
|
|
561
|
+
if (typeof c.id !== "string") continue;
|
|
562
|
+
const name = Array.isArray(c.names) && typeof c.names[0] === "string" ? c.names[0].replace(/^\//, "") : c.id.slice(0, 12);
|
|
563
|
+
const facts = containerFacts(await cliJson(cli, ["containers", "get", c.id]), name);
|
|
564
|
+
if (facts.project === stack) out.push({ name, running: facts.running, state: facts.state, exitCode: facts.exitCode, health: facts.health });
|
|
565
|
+
}
|
|
566
|
+
return out;
|
|
567
|
+
}
|
|
568
|
+
|
|
569
|
+
/** Waits until the stack's sync reports success at `commit`; false when it does not within the limit. */
|
|
570
|
+
async function waitForSync(config: ArcaneConfig, stack: string, commit: string): Promise<{ ok: boolean; last: string }> {
|
|
571
|
+
const deadline = Date.now() + STACK_SYNC_WAIT_MS;
|
|
572
|
+
for (;;) {
|
|
573
|
+
const sync = await withCli(config, async (cli) => projectSync(await cliJson(cli, ["projects", "list", "--all"]), await cliJson(cli, ["gitops", "list", "--all"]), stack));
|
|
574
|
+
const sameCommit = sync.commit !== null && (commit.startsWith(sync.commit) || sync.commit.startsWith(commit));
|
|
575
|
+
if (sameCommit && sync.status === "success") return { ok: true, last: `${sync.status} at ${sync.commit}` };
|
|
576
|
+
if (Date.now() > deadline) return { ok: false, last: `${sync.status ?? "none"} at ${sync.commit ?? "none"}` };
|
|
577
|
+
await new Promise((r) => setTimeout(r, 30_000));
|
|
578
|
+
}
|
|
579
|
+
}
|
|
580
|
+
|
|
581
|
+
async function redeploy(config: ArcaneConfig, stack: string): Promise<string | null> {
|
|
582
|
+
const result = await withCli(config, (cli) => cli(["projects", "redeploy", stack, "--yes"], REDEPLOY_TIMEOUT_MS));
|
|
583
|
+
return result.code === 0 ? null : (result.stderr.trim().split("\n").pop() ?? "").slice(0, 300) || `projects redeploy exited ${result.code}`;
|
|
584
|
+
}
|
|
585
|
+
|
|
586
|
+
/** The containers' problem after a redeploy, polled until none remains or the health wait ends. */
|
|
587
|
+
async function settle(config: ArcaneConfig, stack: string, services: number): Promise<string | null> {
|
|
588
|
+
const deadline = Date.now() + HEALTH_WAIT_MS;
|
|
589
|
+
for (;;) {
|
|
590
|
+
const problem = await withCli(config, async (cli) => {
|
|
591
|
+
const refs = stackServices(await cliJson(cli, ["projects", "list", "--all"]))
|
|
592
|
+
.filter((s) => s.stack === stack)
|
|
593
|
+
.map((s) => s.ref)
|
|
594
|
+
.filter((r): r is string => r !== null);
|
|
595
|
+
return containersProblem(await stackContainers(cli, stack, refs), services);
|
|
596
|
+
});
|
|
597
|
+
if (problem === null || Date.now() > deadline) return problem;
|
|
598
|
+
await new Promise((r) => setTimeout(r, 15_000));
|
|
599
|
+
}
|
|
600
|
+
}
|
|
601
|
+
|
|
602
|
+
/**
|
|
603
|
+
* One stack's pending updates, end to end and without judgment: pinned
|
|
604
|
+
* references move in the Compose file through the GitOps repository, then the
|
|
605
|
+
* stack is redeployed with a pull and checked. Prints one JSON answer whose
|
|
606
|
+
* `status` is updated, rolled back, failed, or nothing to do; the exit code is
|
|
607
|
+
* 0 for all of them so the caller always gets the answer.
|
|
608
|
+
*/
|
|
609
|
+
async function stackUpdate(config: ArcaneConfig, args: string[]): Promise<number> {
|
|
610
|
+
const confirmed = args.includes("--confirm");
|
|
611
|
+
const rest = args.filter((a) => a !== "--confirm");
|
|
612
|
+
if (rest.length !== 1 || !NAME.test(rest[0])) throw new UsageError("stack-update <stack> [--confirm]");
|
|
613
|
+
const stack = rest[0];
|
|
614
|
+
if (stack === "arcane") throw new UsageError("arcane is the manager itself: use manager-upgrade (the Upgrade Arcane button)");
|
|
615
|
+
const spec = repoSpec(config);
|
|
616
|
+
const dir = await ensureCheckout(spec);
|
|
617
|
+
await git(spec, dir, ["pull", "--ff-only", "origin", config.gitopsBranch]);
|
|
618
|
+
const repo = await repoStatus(spec, dir);
|
|
619
|
+
if (repo.changed.length > 0) throw new ServiceError(`the arcane checkout has uncommitted changes: ${repo.changed.join(", ")}`);
|
|
620
|
+
if (repo.ahead > 0) throw new ServiceError(`the arcane checkout is ${repo.ahead} commit(s) ahead of origin`);
|
|
621
|
+
const holdsPath = join(dir, HOLDS_FILE);
|
|
622
|
+
const holds = existsSync(holdsPath) ? parseHolds(readFileSync(holdsPath, "utf-8")) : {};
|
|
623
|
+
if (holds[stack]) throw new UsageError(`${stack} is held in ${HOLDS_FILE}: ${holds[stack]}`);
|
|
624
|
+
|
|
625
|
+
const before = await withCli(config, (cli) => stackState(cli, stack, Date.now()));
|
|
626
|
+
if (before.stopped) throw new UsageError(`${stack} is stopped; a redeploy would start it, so start it on purpose first`);
|
|
627
|
+
const pending = pendingRows(before.rows);
|
|
628
|
+
const answer = (status: StackStatus, extra: Record<string, unknown>) => printJson({ status, stack, ...extra });
|
|
629
|
+
if (pending.length === 0) {
|
|
630
|
+
answer("nothing to do", { unknown: before.rows.filter((r) => r.status === "unknown").map((r) => ({ service: r.service, note: r.note })) });
|
|
631
|
+
return 0;
|
|
632
|
+
}
|
|
633
|
+
const majors = pending.filter((r) => crossesMajor(r.running, r.latest));
|
|
634
|
+
if (majors.length > 0 && !confirmed) {
|
|
635
|
+
throw new UsageError(`${majors.map((r) => `${r.service} ${r.running} -> ${r.latest}`).join(", ")} crosses a major version: summarise release-notes ${stack}/<service> for the user, then repeat with --confirm after their go`);
|
|
636
|
+
}
|
|
637
|
+
const results: ServiceResult[] = pending.map((r) => ({ service: r.service, ref: r.ref, from: r.running, to: r.latest, applied: nextImageRef(r) ?? "pull", result: "unchecked" }));
|
|
638
|
+
const edits = results.filter((r) => r.applied !== "pull");
|
|
639
|
+
|
|
640
|
+
let commit: string | null = null;
|
|
641
|
+
if (edits.length > 0) {
|
|
642
|
+
if (before.composePath === null) throw new ServiceError(`${stack} has no single GitOps sync with a Compose path`);
|
|
643
|
+
const file = safeRepoPath(dir, before.composePath);
|
|
644
|
+
let text = readFileSync(file, "utf-8");
|
|
645
|
+
const moves = new Map(edits.map((e) => [e.ref, e.applied]));
|
|
646
|
+
for (const [from, to] of moves) text = replaceImageRef(text, from, to);
|
|
647
|
+
writeFileSync(file, text);
|
|
648
|
+
await git(spec, dir, ["add", before.composePath]);
|
|
649
|
+
await git(spec, dir, ["commit", "--quiet", "-m", commitMessage(stack, edits.map((e) => ({ service: e.service, from: e.ref, to: e.applied })))]);
|
|
650
|
+
commit = (await git(spec, dir, ["rev-parse", "HEAD"])).trim();
|
|
651
|
+
await git(spec, dir, ["push", "origin", config.gitopsBranch]);
|
|
652
|
+
const synced = await waitForSync(config, stack, commit);
|
|
653
|
+
if (!synced.ok) {
|
|
654
|
+
await git(spec, dir, ["revert", "--no-edit", commit]);
|
|
655
|
+
await git(spec, dir, ["push", "origin", config.gitopsBranch]);
|
|
656
|
+
answer("failed", { reason: `the sync did not report success at ${commit.slice(0, 12)} within ${STACK_SYNC_WAIT_MS / MINUTE} minutes (last: ${synced.last}); the commit was reverted and nothing was redeployed`, commit, services: results });
|
|
657
|
+
return 0;
|
|
658
|
+
}
|
|
659
|
+
}
|
|
660
|
+
|
|
661
|
+
let problem = await redeploy(config, stack);
|
|
662
|
+
if (problem === null) problem = await settle(config, stack, before.services);
|
|
663
|
+
if (problem === null) {
|
|
664
|
+
const after = await withCli(config, (cli) => stackState(cli, stack, Date.now()));
|
|
665
|
+
for (const r of results) {
|
|
666
|
+
const now = after.rows.find((x) => x.service === r.service);
|
|
667
|
+
// Current when nothing newer is pending, or when the target runs and a release appeared meanwhile.
|
|
668
|
+
r.result = now !== undefined && (now.status === "up-to-date" || (now.status !== "unknown" && now.running === r.to)) ? "current" : "behind";
|
|
669
|
+
}
|
|
670
|
+
const behind = results.filter((r) => r.result === "behind").map((r) => r.service);
|
|
671
|
+
if (behind.length > 0) problem = `still behind after the redeploy: ${behind.join(", ")}`;
|
|
672
|
+
}
|
|
673
|
+
if (problem === null) {
|
|
674
|
+
answer("updated", { services: results, ...(commit ? { commit } : {}) });
|
|
675
|
+
return 0;
|
|
676
|
+
}
|
|
677
|
+
if (commit === null) {
|
|
678
|
+
answer("failed", { reason: problem, services: results, note: "the stack was redeployed with pulled images; a pull cannot be rolled back, so check the stack with the arcane specialist" });
|
|
679
|
+
return 0;
|
|
680
|
+
}
|
|
681
|
+
await git(spec, dir, ["revert", "--no-edit", commit]);
|
|
682
|
+
await git(spec, dir, ["push", "origin", config.gitopsBranch]);
|
|
683
|
+
const revert = (await git(spec, dir, ["rev-parse", "HEAD"])).trim();
|
|
684
|
+
const synced = await waitForSync(config, stack, revert);
|
|
685
|
+
const again = synced.ok ? await redeploy(config, stack) : `the revert's sync did not report success (last: ${synced.last})`;
|
|
686
|
+
const settled = again === null ? await settle(config, stack, before.services) : again;
|
|
687
|
+
if (settled === null) {
|
|
688
|
+
answer("rolled back", { reason: problem, services: results, commit, revert, note: "the previous Compose file runs again; images pulled for floating tags stay pulled" });
|
|
689
|
+
return 0;
|
|
690
|
+
}
|
|
691
|
+
answer("failed", { reason: `${problem}; the rollback did not finish either: ${settled}`, services: results, commit, revert });
|
|
692
|
+
return 0;
|
|
693
|
+
}
|
|
694
|
+
|
|
483
695
|
export async function command(args: string[]): Promise<number> {
|
|
484
696
|
if (args.length === 0 || args[0] === "--help" || args[0] === "help") {
|
|
485
697
|
printRaw(HELP);
|
|
486
698
|
return 0;
|
|
487
699
|
}
|
|
488
700
|
const config = readConfig<ArcaneConfig>(SERVICE, SHAPE);
|
|
489
|
-
if (args[0] === "git") return repoCommand(repoSpec(config), args.slice(1));
|
|
701
|
+
if (args[0] === "git") return checkoutLocked(() => repoCommand(repoSpec(config), args.slice(1)));
|
|
490
702
|
if (args[0] === "update-report") return updateReport(config, args.slice(1));
|
|
491
703
|
if (args[0] === "release-notes") return releaseNotes(config, args.slice(1));
|
|
492
704
|
if (args[0] === "host") return hostCommand(config, args.slice(1));
|
|
493
705
|
if (args[0] === "project-status") return projectStatus(config, args.slice(1));
|
|
494
706
|
if (args[0] === "deployment") return deployment(config, args.slice(1));
|
|
495
|
-
if (args[0] === "manager-upgrade") return managerUpgrade(config, args.slice(1));
|
|
707
|
+
if (args[0] === "manager-upgrade") return checkoutLocked(() => managerUpgrade(config, args.slice(1)));
|
|
496
708
|
if (args[0] === "manager-rollback") return managerRollback(args.slice(1));
|
|
709
|
+
if (args[0] === "stack-update") return checkoutLocked(() => stackUpdate(config, args.slice(1)));
|
|
497
710
|
return passthrough(config, args);
|
|
498
711
|
}
|
|
499
712
|
|
|
@@ -0,0 +1,101 @@
|
|
|
1
|
+
import { describe, expect, it } from "vitest";
|
|
2
|
+
import type { UpdateRow } from "./image-updates.ts";
|
|
3
|
+
import { commitMessage, containersProblem, crossesMajor, nextImageRef, parseHolds, pendingRows, replaceImageRef, type StackContainer } from "./stack-update.ts";
|
|
4
|
+
|
|
5
|
+
const A = `sha256:${"a".repeat(64)}`;
|
|
6
|
+
const B = `sha256:${"b".repeat(64)}`;
|
|
7
|
+
|
|
8
|
+
function row(over: Partial<UpdateRow>): UpdateRow {
|
|
9
|
+
return { stack: "s", service: "svc", ref: "nginx:1.31.4-alpine", image: "nginx:1.31.4-alpine", kind: "fixed", running: "1.31.4-alpine", latest: "1.31.6-alpine", status: "release", action: "edit tag", note: "", ...over };
|
|
10
|
+
}
|
|
11
|
+
|
|
12
|
+
describe("update holds", () => {
|
|
13
|
+
it("reads stack names with a reason", () => {
|
|
14
|
+
expect(parseHolds('{"supabase": " coordinated release: review first "}')).toEqual({ supabase: "coordinated release: review first" });
|
|
15
|
+
expect(parseHolds("{}")).toEqual({});
|
|
16
|
+
});
|
|
17
|
+
|
|
18
|
+
it("refuses a malformed file instead of holding nothing", () => {
|
|
19
|
+
expect(() => parseHolds("not json")).toThrow(/not JSON/);
|
|
20
|
+
expect(() => parseHolds('["supabase"]')).toThrow(/object of stack name/);
|
|
21
|
+
expect(() => parseHolds('{"supabase": ""}')).toThrow(/non-empty reason/);
|
|
22
|
+
expect(() => parseHolds('{"../x": "why"}')).toThrow(/stack name/);
|
|
23
|
+
});
|
|
24
|
+
});
|
|
25
|
+
|
|
26
|
+
describe("pending updates", () => {
|
|
27
|
+
it("keeps the rows an update changes and spots a major version", () => {
|
|
28
|
+
const rows = [row({ status: "update" }), row({ status: "release" }), row({ status: "rebuild" }), row({ status: "up-to-date" }), row({ status: "unknown" })];
|
|
29
|
+
expect(pendingRows(rows).map((r) => r.status)).toEqual(["update", "release", "rebuild"]);
|
|
30
|
+
expect(crossesMajor("2.14.3", "3.0.0")).toBe(true);
|
|
31
|
+
expect(crossesMajor("v2.16.1", "v2.17.0")).toBe(false);
|
|
32
|
+
expect(crossesMajor("older build", "3.0.0")).toBe(false);
|
|
33
|
+
});
|
|
34
|
+
|
|
35
|
+
it("moves a pinned tag or digest and leaves pulls to the redeploy", () => {
|
|
36
|
+
expect(nextImageRef(row({}))).toBe("nginx:1.31.6-alpine");
|
|
37
|
+
expect(nextImageRef(row({ ref: "registry.local:5000/app:1.2", latest: "1.3" }))).toBe("registry.local:5000/app:1.3");
|
|
38
|
+
expect(nextImageRef(row({ ref: `ghcr.io/x/y@${A}`, action: "edit digest", latestDigest: B }))).toBe(`ghcr.io/x/y@${B}`);
|
|
39
|
+
expect(nextImageRef(row({ ref: `ghcr.io/x/y:v1@${A}`, action: "edit digest", latestDigest: B }))).toBe(`ghcr.io/x/y:v1@${B}`);
|
|
40
|
+
expect(nextImageRef(row({ action: "pull" }))).toBeNull();
|
|
41
|
+
expect(() => nextImageRef(row({ latest: "new build" }))).toThrow(/cannot move/);
|
|
42
|
+
expect(() => nextImageRef(row({ ref: `ghcr.io/x/y@${A}`, action: "edit digest", latestDigest: "sha256:short" }))).toThrow(/new digest/);
|
|
43
|
+
});
|
|
44
|
+
});
|
|
45
|
+
|
|
46
|
+
describe("Compose rewrite", () => {
|
|
47
|
+
const compose = [
|
|
48
|
+
"services:",
|
|
49
|
+
" db:",
|
|
50
|
+
" image: postgres:18",
|
|
51
|
+
" quoted:",
|
|
52
|
+
' image: "postgres:18" # pinned on purpose',
|
|
53
|
+
" single:",
|
|
54
|
+
" image: 'postgres:18'",
|
|
55
|
+
" longer:",
|
|
56
|
+
" image: postgres:18.1",
|
|
57
|
+
" other:",
|
|
58
|
+
" image: mypostgres:18",
|
|
59
|
+
" mixed:",
|
|
60
|
+
" image: \"postgres:18'",
|
|
61
|
+
" env:",
|
|
62
|
+
" environment:",
|
|
63
|
+
" IMAGE: postgres:18",
|
|
64
|
+
" image_note: image: postgres:18",
|
|
65
|
+
"",
|
|
66
|
+
].join("\n");
|
|
67
|
+
|
|
68
|
+
it("changes only image lines that name exactly the reference", () => {
|
|
69
|
+
const next = replaceImageRef(compose, "postgres:18", "postgres:19").split("\n");
|
|
70
|
+
expect(next[2]).toBe(" image: postgres:19");
|
|
71
|
+
expect(next[4]).toBe(' image: "postgres:19" # pinned on purpose');
|
|
72
|
+
expect(next[6]).toBe(" image: 'postgres:19'");
|
|
73
|
+
expect(next.slice(7)).toEqual(compose.split("\n").slice(7));
|
|
74
|
+
});
|
|
75
|
+
|
|
76
|
+
it("treats the reference as text, not a pattern, and refuses a file without it", () => {
|
|
77
|
+
expect(replaceImageRef(" image: a.b:1+x\n image: aXb:1+x\n", "a.b:1+x", "a.b:2")).toBe(" image: a.b:2\n image: aXb:1+x\n");
|
|
78
|
+
expect(() => replaceImageRef(compose, "postgres:17", "postgres:18")).toThrow(/no "image: postgres:17" line/);
|
|
79
|
+
});
|
|
80
|
+
});
|
|
81
|
+
|
|
82
|
+
describe("stack health after a redeploy", () => {
|
|
83
|
+
const up = (name: string, over: Partial<StackContainer> = {}): StackContainer => ({ name, running: true, state: "running", exitCode: null, health: "healthy", ...over });
|
|
84
|
+
|
|
85
|
+
it("passes running containers and finished one-shot jobs", () => {
|
|
86
|
+
expect(containersProblem([up("db"), up("app", { health: "none" }), up("init", { running: false, state: "exited", exitCode: 0 })], 3)).toBeNull();
|
|
87
|
+
});
|
|
88
|
+
|
|
89
|
+
it("names missing, stopped, unhealthy, and starting containers", () => {
|
|
90
|
+
expect(containersProblem([up("db")], 2)).toBe("1 of 2 services have a container");
|
|
91
|
+
expect(containersProblem([up("db"), up("init", { running: false, state: "exited", exitCode: 1 })], 2)).toBe("not running: init");
|
|
92
|
+
expect(containersProblem([up("db", { health: "unhealthy" }), up("app", { health: "starting" })], 2)).toBe("health: db (unhealthy), app (starting)");
|
|
93
|
+
});
|
|
94
|
+
});
|
|
95
|
+
|
|
96
|
+
describe("commit message", () => {
|
|
97
|
+
it("names the stack and every moved reference on one short line", () => {
|
|
98
|
+
expect(commitMessage("media", [{ service: "sonarr", from: "a:1", to: "a:2" }, { service: "radarr", from: "b:1", to: "b:2" }])).toBe("media: sonarr a:1 -> a:2, radarr b:1 -> b:2");
|
|
99
|
+
expect(commitMessage("s", Array.from({ length: 20 }, () => ({ service: "x", from: "y:1", to: "y:2" }))).length).toBe(200);
|
|
100
|
+
});
|
|
101
|
+
});
|