@astrofoundry/pi-astro 0.40.48 → 0.40.50

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/agents/arcane.md CHANGED
@@ -20,7 +20,7 @@ Rules:
20
20
  - Operational actions (restart, logs, stats, redeploy, image pull, prune) go straight through the CLI.
21
21
  - Confirm destructive operations (`down`, `delete`, `prune`) are explicitly requested in the task before running them.
22
22
  - Updating Arcane itself is `manager-upgrade <version>` (owner approval), which runs the whole documented procedure and rolls back on failed checks; the owner's Upgrade button on the Arcane Daily Report runs the same. Never run `system upgrade`, `projects upgrade arcane`, or a manual sync of `arcane`, and never hand host commands to the caller.
23
- - Host evidence (memory, logs, OOM, capacity, storage, audit) comes from `host <command>`, never from guesses; `host db-restore` only fills an empty database and needs approval.
23
+ - Host evidence (memory, logs, OOM, capacity, storage, audit) comes from `host <command>`, never from guesses; `host db-restore` only fills an empty database and needs approval; `host bigcapital-switch` stops Bigcapital and needs approval.
24
24
  - For update questions run `update-report` and answer with its table verbatim in a code block; never answer from `projects updates`, which is stored and paginated.
25
25
  - Image updates that `update-report` offers go through `stack-update <stack>`, one stack at a time and only when asked; it edits, commits, redeploys, checks, and rolls back on its own. It refuses held stacks (`update-holds.json`) and `arcane`.
26
26
  - Any other image change to one service the caller asks for (a tag switch, a pin, a digest) is a definition change: change only that `image:` line through the GitOps path (`git write`, `git diff`, `git commit`, `git push`, then the stack's `gitops sync`), as the skill describes. It is not a reason to refuse.
@@ -57,7 +57,7 @@ export function edgeStatus(status: string | null, updates: string | null): AreaS
57
57
  const parts = [
58
58
  down.length === 0 ? `${units.length} services active` : `down: ${down.join(", ")}`,
59
59
  `WireGuard ${handshake || "unknown"}`,
60
- rows.get("reboot required") === "yes" ? "reboot pending (it reboots itself at 02:30 UTC)" : "no reboot pending",
60
+ rows.get("reboot required") === "yes" ? "reboot pending (it reboots itself at 01:20 UTC)" : "no reboot pending",
61
61
  upgradable === null ? "updates unknown" : `${upgradable} upgradable package${upgradable === 1 ? "" : "s"}`,
62
62
  ];
63
63
  const tunnel = age === null || age > STALE_HANDSHAKE_S;
@@ -171,7 +171,7 @@ export function guestOwner(name: string): string {
171
171
  return "Proxmox";
172
172
  }
173
173
 
174
- /** Each guest's and Synapse's `updates` after the 04:30 run: anything still waiting did not install and needs a look. */
174
+ /** Each guest's and Synapse's `updates` after the 02:25 run: anything still waiting did not install and needs a look. */
175
175
  export function updatesStatus(hosts: Readonly<Record<string, string | null>>): AreaStatus {
176
176
  const names = Object.keys(hosts);
177
177
  const wazuh = wazuhRelease(hosts);
@@ -319,7 +319,7 @@ export function mediaStatus(sab: unknown): AreaStatus {
319
319
  return { area: "Media", line, ...(warnings > 0 ? { problem: `Media: the downloader shows ${warnings} warning${warnings === 1 ? "" : "s"}`, route: "media" as const } : {}) };
320
320
  }
321
321
 
322
- /** Every guest is backed up nightly at 02:30; an archive older than this means last night's run missed it. */
322
+ /** Every guest is backed up nightly from 02:00; an archive older than this means last night's run missed it. */
323
323
  const MISSED_BACKUP_S = 30 * HOUR;
324
324
 
325
325
  /** `backup vzdump-archives` and `restic-check-status`: last night's Proxmox archives and the weekly restic check. */
@@ -45,6 +45,8 @@ describe("approvals", () => {
45
45
  expect(isRisky("arcane", ["host", "db-restore", "mealie", "20261019T120000Z"])).toBe(true);
46
46
  expect(isRisky("arcane", ["host", "radarr-movie", "tt0120737"])).toBe(true);
47
47
  expect(isRisky("arcane", ["host", "db-dump", "mealie"])).toBe(false);
48
+ expect(isRisky("arcane", ["host", "bigcapital-switch", "20261104T080000Z", "a87df4f"])).toBe(true);
49
+ expect(isRisky("arcane", ["host", "bigcapital-rehearse", "20261104T080000Z", "a87df4f"])).toBe(false);
48
50
  expect(isRisky("arcane", ["host", "container-logs", "sonarr", "2026-09-06T12:00:00Z", "2026-09-06T12:15:00Z"])).toBe(false);
49
51
  expect(isRisky("edge", ["vps", "reboot", "--confirm", "--help"])).toBe(true);
50
52
  expect(isRisky("dns", ["technitium", "primary", "zone-delete", "example.com", "--help"])).toBe(true);
@@ -27,9 +27,9 @@ export const RISKY: Readonly<Record<string, readonly RegExp[]>> = {
27
27
  /^(images|volumes|networks)\s+(prune|remove|rm|delete)\b/,
28
28
  /^system\s+(prune|upgrade)\b/,
29
29
  /^gitops\s+delete\b/,
30
- // Arcane VM: upgrading or rolling back the manager, filling a database, and reading Radarr's key from its host config.
30
+ // Arcane VM: upgrading or rolling back the manager, filling a database, stopping Bigcapital for its switch, and reading Radarr's key from its host config.
31
31
  /^manager-(upgrade|rollback)\b/,
32
- /^host\s+(db-restore|radarr-movie|radarr-interface)\b/,
32
+ /^host\s+(db-restore|bigcapital-switch|radarr-movie|radarr-interface)\b/,
33
33
  ],
34
34
  identity: [/^zitadel\s+(DELETE|POST|PUT|PATCH)\b/, /^dmz\s+(deploy-config|pomerium-restart|pomerium-upgrade|render|lego-renew)\b/],
35
35
  // Every UniFi write, on either console.
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@astrofoundry/pi-astro",
3
- "version": "0.40.48",
3
+ "version": "0.40.50",
4
4
  "description": "Personal pi customizations (extensions, subagents, skills, prompts, themes) for the pi coding agent.",
5
5
  "keywords": [
6
6
  "pi-package"
@@ -88,6 +88,12 @@ The usual trigger is the owner's **Upgrade Arcane** button under the Arcane Dail
88
88
  - `db-list mealie|paperless`, `db-dump mealie|paperless` (gzip pg_dump into `/var/backups/spc-arcane/<stack>/<UTC>`, checked, with row counts), `db-restore <stack> <stamp>` (owner approval): fills an **empty** database from one of those dumps, the app container stopped meanwhile. It refuses a database that holds tables; emptying one is an operator decision. Dumps are never deleted by the tool; `db-list` names those beyond the last three in `over_keep`.
89
89
  - `supabase-backup`: the backup `supabase/AGENTS.md` asks for before a Supabase change. It writes custom-format dumps of `postgres` and `_supabase`, `globals.sql`, and `db-config.tar.gz` (the pgsodium root key) into `/var/backups/supabase-upgrades/<UTC>`, checks each dump with pg_restore's listing and full decoding, and prints the manifest with the image's repo digests (record them as the rollback digest). It refuses when the disk has less free space than the databases. Restoring stays an operator step.
90
90
 
91
+ - Bigcapital base-image upgrades (`bigcapital/README.md` "Base-image upgrades"), all under `/var/backups/bigcapital-upgrades/<UTC>`:
92
+ - `bigcapital-backup`: an online, consistent dump of the system and tenant databases, with exact row counts, each tenant's ledger totals (kept on the host, never printed), and the running image digests.
93
+ - `bigcapital-rehearse <stamp> <build>`: restores that dump into the current MariaDB on an internal Docker network, upgrades it with the build's MariaDB (`MARIADB_AUTO_UPGRADE`), runs the build's migrations, starts its Redis and server, and validates its Envoy configuration. It compares rows and ledgers at each step, removes everything it created, and prints error names only.
94
+ - `bigcapital-switch <stamp> <build>` (owner approval): needs a passed rehearsal of that build and the `astrofoundry` tags pointing at it. It stops the app, takes the quiescent dump, saves Redis, stops both, and copies their volumes to `bigcapital_mysql_<UTC>` and `bigcapital_redis_<UTC>`. The stack stays stopped for the GitOps commit that mounts the copies; a failure starts the stopped containers again. The old volumes stay untouched for a rollback.
95
+ - `bigcapital-verify <stamp>`: after that commit's redeploy, checks the mounted copies, the running build, the migration exit code, server health, and rows and ledgers against the switch's dump.
96
+
91
97
  Public routes (Supabase, Grimoire, Mealie URLs) are `astro.edge`'s `route-probe`, run from Cortex.
92
98
 
93
99
  `["stack-rollback", "<stack>", "<commit>"]` undoes one earlier `stack-update` commit of that stack's Compose file (subject `<stack>: ...`, only that file, on the branch): revert, push, start the sync, redeploy, recreate skipped containers, check. The Discord bridge runs it when a stack fails its check minutes after an automatic update; run it yourself only when the caller asks for that rollback.
@@ -7,12 +7,12 @@ description: How the backup specialist checks and runs the homelab backup layers
7
7
 
8
8
  Four layers protect the homelab. Every command is a fixed operation on Pulsar (root) or on the Arcane VM (user `arcane`); `["--help"]` lists them.
9
9
 
10
- | Layer | What | Schedule | Commands |
10
+ | Layer | What | Schedule (Europe/Madrid) | Commands |
11
11
  |---|---|---|---|
12
- | 1 | vzdump of every guest to PVE storage `unas-backup` (NAS `//10.0.50.50/Shared_Drive/BACKUP/pulsar/dump`), snapshot mode, zstd, 7 daily + 4 weekly | 02:30 | `vzdump-jobs`, `vzdump-archives`, `vzdump-tasks`, `vzdump-run <vmid>` |
13
- | 2 | App-consistent service backups from the Arcane VM into the shared Restic repository (`/mnt/backup/restic` there), one tag per service, 30 daily + 12 monthly | 03:00, check Sunday 04:00 | `arcane-backup-status|run|journal`, `restic-check-status|run|journal` |
14
- | 3 | Pulsar host config, Technitium ZIP exports, Identity `pg_dump` into Restic, tag `pulsar-host` | 03:30 | `host-backup-status|run|journal` |
15
- | 4 | Offsite copy of the Restic repository to `gs://ardilla-backup/pulsar/restic` from LXC 102 (append-only service account) | 04:30 UTC | `gcs-status|run|journal` |
12
+ | 1 | vzdump of every guest to PVE storage `unas-backup` (NAS `//10.0.50.50/Shared_Drive/BACKUP/pulsar/dump`), snapshot mode, zstd, 7 daily + 4 weekly | 02:00 | `vzdump-jobs`, `vzdump-archives`, `vzdump-tasks`, `vzdump-run <vmid>` |
13
+ | 2 | App-consistent service backups from the Arcane VM into the shared Restic repository (`/mnt/backup/restic` there), one tag per service, 30 daily + 12 monthly | 02:15, check Sunday 02:25 | `arcane-backup-status|run|journal`, `restic-check-status|run|journal` |
14
+ | 3 | Pulsar host config, Technitium ZIP exports, Identity `pg_dump` into Restic, tag `pulsar-host` | 02:20 | `host-backup-status|run|journal` |
15
+ | 4 | Offsite copy of the Restic repository to `gs://ardilla-backup/pulsar/restic` from LXC 102 (append-only service account) | 03:10 | `gcs-status|run|journal` |
16
16
 
17
17
  Restic commands run on Pulsar against `/mnt/pve/unas-backup/restic` with the host's password file: `restic-snapshots [tag]`, `restic-stats [tag]`, `restic-ls <snapshot> [path]`, `restic-restore <snapshot> <path>`, `restore-ls`, `restore-clean`. `restic-restore` writes under `/var/tmp/spc-restore/<snapshot>/` on Pulsar and nowhere else.
18
18
 
@@ -45,7 +45,7 @@ A service's backup is confirmed when `arcane-backup-status` shows `Result=succes
45
45
 
46
46
  ## Mira guests
47
47
 
48
- SPACE-X (Mira) writes its own vzdump archives to TESLA (`tesla-backup`, `BACKUP/space-x/dump`), and Pulsar's `mira-backup-pull.timer` copies them to the UNAS (`BACKUP/space-x/dump`) at 05:00 without mirroring Mira's deletions. This tool does not read SPACE-X; the proxmox specialist's `space-x tasks` shows the runs, and Kuma monitors `space-x-backup` and `mira-backup-pull` carry the heartbeats.
48
+ SPACE-X (Mira) writes its own vzdump archives to TESLA (`tesla-backup`, `BACKUP/space-x/dump`), and Pulsar's `mira-backup-pull.timer` copies them to the UNAS (`BACKUP/space-x/dump`) at 03:05 without mirroring Mira's deletions. This tool does not read SPACE-X; the proxmox specialist's `space-x tasks` shows the runs, and Kuma monitors `space-x-backup` and `mira-backup-pull` carry the heartbeats.
49
49
 
50
50
  ## Reading the output
51
51
 
@@ -48,7 +48,7 @@ Removing a hostname is the same with a removed line; check first that no public
48
48
 
49
49
  ## Vulnerability on the VPS
50
50
 
51
- Wazuh reports a CVE against a package (the security specialist passes `package`, installed `version`, and `condition`, the fixed version). The VPS patches itself: unattended-upgrades installs Debian, Debian-Security, and Google guest-agent updates once a day and reboots at 02:30 UTC when a package needs it. It does not take `trixie-updates` or Google's cloud-sdk repository, so packages from those (such as `tzdata` and `google-cloud-cli`) wait for the operator. Answer which of three states applies:
51
+ Wazuh reports a CVE against a package (the security specialist passes `package`, installed `version`, and `condition`, the fixed version). The VPS patches itself: unattended-upgrades installs Debian, Debian-Security, `trixie-updates`, Google guest-agent, Google Cloud CLI, and CrowdSec bouncer updates at 02:05 Europe/Madrid, after a package-list refresh at 02:00, and reboots at 01:20 UTC when a package needs it (`00-frontdoor-vps/system/52unattended-upgrades-frontdoor`). Answer which of three states applies:
52
52
 
53
53
  1. `package <name>`, then `changelog <name>`: if the installed version comes from the security origin and its changelog entry names the CVE, it is patched; say by which upload and, from `auto-upgrades` or the dpkg log, when. Wazuh may keep flagging the new version until its feed learns the fixed version; that is feed lag, not an open hole.
54
54
  2. Otherwise, if the candidate's changelog names the CVE, it is scheduled: `auto-upgrades` shows when `apt-daily-upgrade.timer` runs next. Say so with the time.
@@ -64,7 +64,7 @@ The `security` tool is read-only. Every call is a fixed command on the guest; `[
64
64
 
65
65
  You observe; the owning specialist acts. When a finding concerns a host in the table, call that specialist through `subagent` with the facts (`agent`, `package`, installed `version`, `condition`, `cve`) and a precise question, then report its answer with your own. Do not tell the caller to run commands on a host that has an owner.
66
66
 
67
- Debian guests and Synapse install every stable update each night (04:30 Europe/Madrid, technitium2 04:50, the Mira guests 04:30 Asia/Dubai) and reboot right after when a package needs it; `wazuh-agent` is held. The VPS installs Debian and Debian-Security updates once a day and reboots at 02:30 UTC when needed; packages from `trixie-updates` wait for the operator. The morning run upgrades the Pulsar and SPACE-X hosts and offers the owner a Reboot button. For a vulnerability alert, ask the owner whether the package is already upgraded, will be at the next run (the fix is in the index), or is stuck (the fix is not in the index or the run failed), and to confirm from the package changelog that the installed version names the CVE. Only the stuck case needs the operator; say so plainly, with the versions. Answer with one of: patched (version, when, by what), scheduled (next run time), stuck (why), or feed lag (patched, Wazuh not updated yet).
67
+ Debian guests and Synapse install every stable update each night (02:25 Europe/Madrid, technitium2 02:30, the Mira guests 02:35, each five minutes after a package-list refresh) and reboot right after when a package needs it; `wazuh-agent` is held. The VPS installs Debian, Debian-Security, `trixie-updates`, Google guest-agent, Google Cloud CLI, and CrowdSec bouncer updates at 02:05 Europe/Madrid and reboots at 01:20 UTC when needed. The morning run upgrades the Pulsar and SPACE-X hosts and offers the owner a Reboot button. For a vulnerability alert, ask the owner whether the package is already upgraded, will be at the next run (the fix is in the index), or is stuck (the fix is not in the index or the run failed), and to confirm from the package changelog that the installed version names the CVE. Only the stuck case needs the operator; say so plainly, with the versions. Answer with one of: patched (version, when, by what), scheduled (next run time), stuck (why), or feed lag (patched, Wazuh not updated yet).
68
68
 
69
69
  ## Operator actions (describe, do not perform)
70
70
 
@@ -59,7 +59,7 @@ Wrappers ship as TypeScript and run under Node 24 type stripping: erasable synta
59
59
  - `main()` waits for stdout and stderr to flush before `process.exit`: on a pipe, exiting right after a write drops everything past the 64 KiB buffer, which cut large answers such as `projects list --all` until 2026-09-25 (`lib/flush.test.ts`).
60
60
  - `arcane update-report [--all]` is `lib/image-updates.ts` over `lib/registry.ts`: a fresh `images updates check --all`, then `projects list --all` for the stack and service of each image reference, then the registries' tag lists and manifest digests (anonymous bearer tokens from each registry's challenge; digests by HEAD, which Docker Hub does not count as a pull; lscr.io is read from ghcr.io). A floating or major-series tag is resolved to the release tag whose digest matches the running and the latest build, searching the newest tags of the three commonest release shapes; a fixed tag is compared with the newest tag of the same shape; a digest pin with the newest release. Checks older than one minute before the run count as stale and show as unknown. Private images (anonymous access refused with 401 or 403) are excluded from rows, counts, and table, listed in `privateImages`, and counted in the summary line. The table is fixed-width text for a code block, because Discord renders no Markdown tables. The answer adds `holds` from `update-holds.json` at origin's branch head (`git show`, no working-tree read), or `holdsError` when that file is malformed.
61
61
  - `arcane stack-update <stack> [--confirm]` (`lib/stack-update.ts` for the pure parts) is deterministic: refuses `arcane`, held stacks (the checkout's `update-holds.json` after `pull --ff-only`; a malformed file is an error), a dirty or ahead checkout, stopped stacks, and an update across a major version without `--confirm` (which the gate turns into an owner approval). Pending rows come from a fresh per-image check resolved like `update-report`. `edit tag`/`edit digest` rows rewrite `image:` lines that name exactly the old reference (`replaceImageRef`, quoted or not, trailing comment kept) in the sync's `composePath`, then commit, push, and wait for `gitops list` to report success at that commit (20 min, polled every 30 s). Then `projects redeploy <stack> --yes` (35 min), container checks through `containers list --all` and `containers get` (`composeInfo.projectName`, health; 5 min, polled every 15 s; a job that exited with code 0 counts as done whatever health its image's inherited check left, as bigcapital's migration did on 2026-10-28), and a fresh check that each service is current. A failure after a commit reverts, pushes, waits for the sync, redeploys, and checks again (`rolled back`); a pull-only failure stays `failed`. Every outcome prints JSON with `status` and exits 0. Worst case about 125 minutes, so the gate's `timeoutMs` for arcane must exceed it (8,100,000). It holds the `arcane-gitops` lock, so a second checkout call waits up to 2 minutes and then fails as busy; the bridge also runs one Arcane update at a time.
62
- - `arcane host <command>` runs over the forced-command key of user `spc-arcane` on VM 100. `HOST_COMMANDS` in `lib/arcane-host.ts` must list exactly the operations of `entry/remote/spc-arcane-op.py` (a test compares them); the tracked copies live in the arcane repository's `host/` (`spc-arcane-host-entry`, `spc-arcane-op`, `spc-arcane-sudoers`, `spc-arcane-install.sh`) with the probes in `host/spc-arcane-probes/`. The entry only splits words (letters, digits, space, `._:/+-`) and hands them to `sudo -n /usr/local/sbin/spc-arcane-op`, the one command its sudoers line allows; the op checks the name and every argument and runs the reviewed arcane-ops probes, which print only allowlisted fields. Manager steps (`manager-*`) are internal to `manager-upgrade`/`manager-rollback` and refused through `host`. Nothing on the host deletes data: a manager rollback restores the image and Compose file, not the data volume (R3 stays manual), `db-restore` only fills an empty database, and dumps are never pruned by the tool. `supabase-backup` writes the pre-change backup of `supabase/AGENTS.md` (both databases with `pg_dump -Fc`, each listed and fully decoded by `pg_restore`, `pg_dumpall --globals-only`, and the `supabase-db-config` volume, which must hold `pgsodium_root.key`) into `/var/backups/supabase-upgrades/<UTC>`, the folder of the earlier manual backups. `host-audit` compares the host's hashes with `host/<name>` of the `arcane-gitops` checkout after a pull.
62
+ - `arcane host <command>` runs over the forced-command key of user `spc-arcane` on VM 100. `HOST_COMMANDS` in `lib/arcane-host.ts` must list exactly the operations of `entry/remote/spc-arcane-op.py` (a test compares them); the tracked copies live in the arcane repository's `host/` (`spc-arcane-host-entry`, `spc-arcane-op`, `spc-arcane-sudoers`, `spc-arcane-install.sh`) with the probes in `host/spc-arcane-probes/`. The entry only splits words (letters, digits, space, `._:/+-`) and hands them to `sudo -n /usr/local/sbin/spc-arcane-op`, the one command its sudoers line allows; the op checks the name and every argument and runs the reviewed arcane-ops probes, which print only allowlisted fields. Manager steps (`manager-*`) are internal to `manager-upgrade`/`manager-rollback` and refused through `host`. Nothing on the host deletes data: a manager rollback restores the image and Compose file, not the data volume (R3 stays manual), `db-restore` only fills an empty database, and dumps are never pruned by the tool. `supabase-backup` writes the pre-change backup of `supabase/AGENTS.md` (both databases with `pg_dump -Fc`, each listed and fully decoded by `pg_restore`, `pg_dumpall --globals-only`, and the `supabase-db-config` volume, which must hold `pgsodium_root.key`) into `/var/backups/supabase-upgrades/<UTC>`, the folder of the earlier manual backups. `host-audit` compares the host's hashes with `host/<name>` of the `arcane-gitops` checkout after a pull. The `bigcapital-*` commands run the base-image upgrade of `bigcapital/README.md`: an online backup, a rehearsal on an internal Docker network (restore into the current MariaDB, upgrade with the build's, migrations, server health, Envoy validation; everything it created removed afterwards), the owner-approved switch (stop, quiescent dump, Redis `SAVE`, volume copies compared entry by entry, the stack left stopped for the GitOps commit, the old volumes untouched), and a verify against the switch's row counts and ledger totals.
63
63
  - `arcane manager-upgrade <version>` (`lib/arcane-manager.ts`) follows arcane/AGENTS.md "Manager upgrades": the official image's revision label must equal the release tag's commit (GitHub API), `manager-prepare` records `compose.before.yaml` and every container before the pin commit, the pin goes through the `arcane-gitops` checkout (`git` is exported from `lib/repo.ts` for this), the wait polls `gitops list` and the deployed Compose hash every 30 s for up to 30 minutes, then pull, cold backup, `up -d --no-deps --pull never`, and checks. Any failure before the restart reverts the pin (`stopped`); a failed check after it reverts the pin and runs `manager-rollback` (`rolled back`). The answer always exits 0 with `status`, so the bridge and the agent both see it.
64
64
  - `media` runs trendarr unchanged: `trendarrCli` in `media.json` is its `dist/cli.js`, started with `process.execPath` because sudo's PATH has no node. Each call gets `HOME` set to a fresh `0700` directory under `~/.specialists/work/media/` that holds only `.trendarr/config.json` (`0600`, trendarr's own schema, built from the note fields and `media.json`) and is removed in `finally`; trendarr reads no environment variables and has no config flag, and Node's `os.homedir()` follows `$HOME` on POSIX. Every call passes `--json`; `init` is refused. An add prints trendarr's per-ID `results` and exits 0 even when trendarr exits 1 for a failed ID, because the gate shows only stderr on failure; other failures carry trendarr's stderr without its `warning:` lines, which go into the JSON as `warnings`. The bridge's slash commands and Add buttons call the same wrapper through the entry point (`extensions/astro-discord/media.ts`).
65
65
  - `arcane release-notes <stack>[/<service>]` is `lib/releases.ts`: a single-image check and `resolveRef` give the running and latest versions, `HttpRegistry.labels` reads `org.opencontainers.image.source` from the latest image (index to linux/amd64 manifest to config blob), and the anonymous GitHub releases API (60 requests an hour, up to 3 pages of 100) gives the releases in between, oldest first. `condenseNotes` keeps each release under 12,000 characters by section priority; `capTotal` drops the oldest releases past 40,000 characters in total.
@@ -16,7 +16,9 @@ import importlib.util
16
16
  import json
17
17
  import os
18
18
  import re
19
+ import secrets
19
20
  import shutil
21
+ import stat
20
22
  import subprocess
21
23
  import sys
22
24
  import tarfile
@@ -56,6 +58,33 @@ SUPABASE_CONFIG = Path("/var/lib/docker/volumes/supabase-db-config/_data")
56
58
  SUPABASE_KEY = "./pgsodium_root.key"
57
59
  SUPABASE_BACKUPS = Path("/var/backups/supabase-upgrades")
58
60
 
61
+ # Bigcapital base-image upgrades (bigcapital/README.md "Update the app"): the
62
+ # Compose containers, the fork's images, and the earlier manual backups' folder.
63
+ BIGCAPITAL = "bigcapital"
64
+ BIGCAPITAL_DB = "bigcapital-mysql-1"
65
+ BIGCAPITAL_REDIS = "bigcapital-redis-1"
66
+ # Stopped first, started last: nothing writes while the database is dumped and copied.
67
+ BIGCAPITAL_APP = ("bigcapital-proxy-1", "bigcapital-webapp-1", "bigcapital-server-1")
68
+ BIGCAPITAL_IMAGE = "ghcr.io/astrofoundry/bigcapital-{}:{}"
69
+ BIGCAPITAL_SERVICES = ("mysql", "redis", "server", "webapp", "proxy", "migration")
70
+ BIGCAPITAL_BACKUPS = Path("/var/backups/bigcapital-upgrades")
71
+ BIGCAPITAL_DUMP = "bigcapital-databases.sql.gz"
72
+ BIGCAPITAL_SYSTEM_DB = "bigcapital_system"
73
+ BIGCAPITAL_TENANT_PREFIX = "bigcapital_tenant_"
74
+ # The server's non-secret settings from bigcapital/compose.yaml, without mail and
75
+ # public URLs; the integrations stay explicit and empty as there.
76
+ BIGCAPITAL_SERVER_ENV = {
77
+ "DB_HOST": "mysql", "DB_USER": "bigcapital", "DB_CHARSET": "utf8", "SYSTEM_DB_NAME": BIGCAPITAL_SYSTEM_DB,
78
+ "TENANT_DB_NAME_PERFIX": BIGCAPITAL_TENANT_PREFIX, "REDIS_HOST": "redis", "REDIS_PORT": "6379", "QUEUE_HOST": "redis",
79
+ "QUEUE_PORT": "6379", "SIGNUP_DISABLED": "false", "SIGNUP_EMAIL_CONFIRMATION": "false", "EXCHANGE_RATE_SERVICE": "open-exchange-rate", "CLICKHOUSE_ENABLED": "false",
80
+ "PLAID_ENV": "sandbox", "S3_REGION": "US", **{name: "" for name in (
81
+ "SIGNUP_ALLOWED_DOMAINS", "SIGNUP_ALLOWED_EMAILS", "OPEN_EXCHANGE_RATE_APP_ID", "BANK_FEED_ENABLED", "PLAID_CLIENT_ID", "PLAID_SECRET", "PLAID_LINK_WEBHOOK",
82
+ "LEMONSQUEEZY_API_KEY", "LEMONSQUEEZY_STORE_ID", "LEMONSQUEEZY_WEBHOOK_SECRET", "HOSTED_ON_BIGCAPITAL_CLOUD",
83
+ "S3_ACCESS_KEY_ID", "S3_SECRET_ACCESS_KEY", "S3_ENDPOINT", "S3_BUCKET", "STRIPE_PAYMENT_SECRET_KEY",
84
+ "STRIPE_PAYMENT_PUBLISHABLE_KEY", "STRIPE_PAYMENT_CLIENT_ID", "STRIPE_PAYMENT_WEBHOOKS_SECRET", "STRIPE_PAYMENT_REDIRECT_URL")},
85
+ }
86
+ SQL_CLIENT = 'exec "$(command -v mariadb || command -v mysql)" -uroot -p"$MYSQL_ROOT_PASSWORD"'
87
+
59
88
  # Tracked host files the audit compares with the repository's host/ folder.
60
89
  HOST_FILES = (
61
90
  ("arcane-backup.sh", "/usr/local/bin/arcane-backup.sh"),
@@ -102,6 +131,8 @@ VERSION = re.compile(r"^v?[0-9]{1,4}\.[0-9]{1,4}\.[0-9]{1,4}$")
102
131
  IMDB = re.compile(r"^tt[0-9]{1,12}$")
103
132
  UUID = re.compile(r"^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$")
104
133
  INTEGER = re.compile(r"^[0-9]{1,9}$")
134
+ BUILD = re.compile(r"^[0-9a-f]{7}$")
135
+ SQL_NAME = re.compile(r"^[A-Za-z0-9_]{1,64}$")
105
136
  OBJECT_PATH = re.compile(r"^[A-Za-z0-9_][A-Za-z0-9_./-]{0,255}$")
106
137
 
107
138
 
@@ -716,6 +747,424 @@ def op_supabase_backup(args):
716
747
  emit(info)
717
748
 
718
749
 
750
+ # ---------------------------------------------------------------- bigcapital base upgrades
751
+
752
+
753
+ def maria(container, statement, timeout=600):
754
+ """One statement through the container's own client as root; rows come back as lists of fields."""
755
+ result = run([DOCKER, "exec", container, "sh", "-c", SQL_CLIENT + ' -N -B -e "$1"', "sh", statement], timeout)
756
+ if result.returncode != 0:
757
+ raise Failed(f"SQL in {container} failed (exit {result.returncode}); its output was withheld")
758
+ return [line.split("\t") for line in result.stdout.decode("utf-8", "replace").splitlines()]
759
+
760
+
761
+ def wait_for_server(container, seconds):
762
+ """A TCP ping: the entrypoint's temporary init and upgrade servers listen only on the socket, so they never count."""
763
+ ping = 'exec "$(command -v mariadb-admin || command -v mysqladmin)" ping -h127.0.0.1 --protocol=tcp'
764
+ deadline = time.monotonic() + seconds
765
+ while time.monotonic() < deadline:
766
+ if run([DOCKER, "exec", container, "sh", "-c", ping], 30).returncode == 0:
767
+ return
768
+ if not container_running(container):
769
+ raise Failed(f"{container} stopped before its server accepted connections")
770
+ time.sleep(3)
771
+ raise Failed(f"{container} did not accept connections within {seconds} s")
772
+
773
+
774
+ def bigcapital_schemas(container):
775
+ pattern = BIGCAPITAL_TENANT_PREFIX.replace("_", "\\_") + "%"
776
+ rows = maria(container, f"SELECT schema_name FROM information_schema.schemata WHERE schema_name = '{BIGCAPITAL_SYSTEM_DB}' "
777
+ f"OR schema_name LIKE '{pattern}' ORDER BY schema_name")
778
+ names = [row[0] for row in rows]
779
+ if BIGCAPITAL_SYSTEM_DB not in names or not all(SQL_NAME.fullmatch(name) for name in names):
780
+ raise Failed("the Bigcapital databases are missing or carry unexpected names")
781
+ return names
782
+
783
+
784
+ def fingerprint(container):
785
+ """Exact row counts of every Bigcapital table and each tenant's ledger totals; every step compares them."""
786
+ schemas = bigcapital_schemas(container)
787
+ listed = ", ".join(f"'{name}'" for name in schemas)
788
+ tables = maria(container, "SELECT table_schema, table_name FROM information_schema.tables "
789
+ f"WHERE table_type = 'BASE TABLE' AND table_schema IN ({listed}) ORDER BY 1, 2")
790
+ if not tables or not all(len(t) == 2 and SQL_NAME.fullmatch(t[1]) for t in tables):
791
+ raise Failed("unexpected table names in the Bigcapital databases")
792
+ union = " UNION ALL ".join(f"SELECT '{s}.{t}', COUNT(*) FROM `{s}`.`{t}`" for s, t in tables)
793
+ rows = {name: int(count) for name, count in maria(container, union, 1800)}
794
+ ledgers = {}
795
+ # Bigcapital's knex mappers upper-case its table names (ACCOUNTS_TRANSACTIONS), but not knex_migrations.
796
+ for schema, table in tables:
797
+ if table.lower() == "accounts_transactions":
798
+ [[credit, debit]] = maria(container, f"SELECT COALESCE(SUM(credit), 0), COALESCE(SUM(debit), 0) FROM `{schema}`.`{table}`")
799
+ ledgers[schema] = [credit, debit]
800
+ return {"schemas": schemas, "rows": rows, "ledgers": ledgers}
801
+
802
+
803
+ def summary(prints):
804
+ """Counts only: the ledger totals stay in the root-only record."""
805
+ rows = prints["rows"]
806
+ tenant = [count for name, count in rows.items() if name.startswith(BIGCAPITAL_TENANT_PREFIX) and name.endswith(".knex_migrations")]
807
+ return {"databases": len(prints["schemas"]), "tables": len(rows), "rows": sum(rows.values()), "ledgers": len(prints["ledgers"]),
808
+ "migrations": {"system": rows.get(f"{BIGCAPITAL_SYSTEM_DB}.knex_migrations"), "tenant": sum(tenant)}}
809
+
810
+
811
+ def differences(before, after):
812
+ def changed(key):
813
+ old, new = before[key], after[key]
814
+ return sorted(name for name in set(old) | set(new) if old.get(name) != new.get(name))
815
+ return {"tables": changed("rows"), "ledgers": changed("ledgers")}
816
+
817
+
818
+ def write_json(path, value):
819
+ write_private(path, json.dumps(value, sort_keys=True, indent=1).encode())
820
+
821
+
822
+ def new_record():
823
+ root_dir(BIGCAPITAL_BACKUPS)
824
+ stamp = utc_stamp()
825
+ path = BIGCAPITAL_BACKUPS / stamp
826
+ if path.exists():
827
+ raise Failed(f"{path} already exists")
828
+ return stamp, root_dir(path)
829
+
830
+
831
+ def existing_record(stamp, usage):
832
+ path = BIGCAPITAL_BACKUPS / need(STAMP, stamp, usage)
833
+ if path.is_symlink() or not path.is_dir():
834
+ raise Failed(f"no Bigcapital record {stamp}")
835
+ return path
836
+
837
+
838
+ def bigcapital_images():
839
+ """Every container of the Compose project with its configured image, image ID and registry digests."""
840
+ names = must([DOCKER, "ps", "-a", "--filter", f"label=com.docker.compose.project={BIGCAPITAL}", "--format", "{{.Names}}"],
841
+ 30, "docker ps").split()
842
+ images = []
843
+ for name in sorted(names):
844
+ image, image_id = must([DOCKER, "inspect", "--format", "{{.Config.Image}} {{.Image}}", name], 30, "docker inspect").split()
845
+ digests = json.loads(must([DOCKER, "image", "inspect", "--format", "{{json .RepoDigests}}", image_id], 30, "docker image inspect"))
846
+ images.append({"container": name, "image": image, "image_id": image_id, "repo_digests": digests})
847
+ return images
848
+
849
+
850
+ def bigcapital_dump(path, schemas):
851
+ """A consistent dump of the Bigcapital databases only: the system schema differs between server versions."""
852
+ target = path / BIGCAPITAL_DUMP
853
+ dump = ('exec "$(command -v mariadb-dump || command -v mysqldump)" -uroot -p"$MYSQL_ROOT_PASSWORD" '
854
+ '--single-transaction --routines --triggers --events --databases "$@"')
855
+ descriptor = os.open(target, os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_NOFOLLOW, 0o600)
856
+ with os.fdopen(descriptor, "wb") as raw, gzip.GzipFile(fileobj=raw, mode="wb") as out:
857
+ process = subprocess.Popen([DOCKER, "exec", BIGCAPITAL_DB, "sh", "-c", dump, "sh", *schemas],
858
+ stdout=subprocess.PIPE, stderr=subprocess.DEVNULL)
859
+ shutil.copyfileobj(process.stdout, out)
860
+ code = process.wait(timeout=3600)
861
+ if code != 0:
862
+ raise Failed(f"the dump failed (exit {code}); {target} is incomplete and has no manifest")
863
+ tail = b""
864
+ with gzip.open(target, "rb") as check:
865
+ for block in iter(lambda: check.read(1 << 20), b""):
866
+ tail = (tail + block)[-4096:]
867
+ if b"-- Dump completed" not in tail:
868
+ raise Failed(f"{target.name} lacks the dump's completion line; it has no manifest")
869
+ return target
870
+
871
+
872
+ def bigcapital_record(stamp, path, quiescent):
873
+ schemas = bigcapital_schemas(BIGCAPITAL_DB)
874
+ listed = ", ".join(f"'{name}'" for name in schemas)
875
+ [[size]] = maria(BIGCAPITAL_DB, f"SELECT COALESCE(SUM(data_length + index_length), 0) FROM information_schema.tables WHERE table_schema IN ({listed})")
876
+ if shutil.disk_usage(path).free < int(size):
877
+ raise Failed(f"less free space under {BIGCAPITAL_BACKUPS} than the databases' {size} bytes; nothing was dumped")
878
+ images = bigcapital_images()
879
+ write_json(path / "release-images.json", images)
880
+ prints = fingerprint(BIGCAPITAL_DB)
881
+ target = bigcapital_dump(path, schemas)
882
+ checksum = sha256_file(target)
883
+ write_private(path / "SHA256SUMS", f"{checksum} {target.name}\n".encode())
884
+ write_json(path / "fingerprint.json", prints)
885
+ info = {"stamp": stamp, "path": str(path), "file": target.name, "bytes": target.stat().st_size, "sha256": checksum,
886
+ "server_version": maria(BIGCAPITAL_DB, "SELECT VERSION()")[0][0], "quiescent": quiescent, "fingerprint": summary(prints),
887
+ "images": {i["container"]: i["image_id"] for i in images}}
888
+ write_json(path / "manifest.json", info)
889
+ return info
890
+
891
+
892
+ def op_bigcapital_backup(args):
893
+ """Online and consistent (--single-transaction) while the app runs; the switch takes its own with the app stopped."""
894
+ arity(args, 0, 0, "bigcapital-backup")
895
+ if not container_running(BIGCAPITAL_DB):
896
+ raise Failed(f"{BIGCAPITAL_DB} is not running")
897
+ stamp, path = new_record()
898
+ emit(bigcapital_record(stamp, path, quiescent=False))
899
+
900
+
901
+ def pull_build(tag):
902
+ """The image ID of each fork image at that tag; the rehearsal and the switch compare them."""
903
+ ids = {}
904
+ for service in BIGCAPITAL_SERVICES:
905
+ ref = BIGCAPITAL_IMAGE.format(service, tag)
906
+ must([DOCKER, "pull", "--quiet", ref], 900, f"docker pull {ref}")
907
+ ids[service] = must([DOCKER, "image", "inspect", "--format", "{{.Id}}", ref], 30, "docker image inspect").strip()
908
+ return ids
909
+
910
+
911
+ def restore_dump(container, source):
912
+ with gzip.open(source, "rb") as dump:
913
+ process = subprocess.Popen([DOCKER, "exec", "-i", container, "sh", "-c", SQL_CLIENT],
914
+ stdin=subprocess.PIPE, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
915
+ try:
916
+ shutil.copyfileobj(dump, process.stdin)
917
+ process.stdin.close()
918
+ except BrokenPipeError:
919
+ pass
920
+ code = process.wait(timeout=3600)
921
+ if code != 0:
922
+ raise Failed(f"the restore exited {code}; its output was withheld")
923
+
924
+
925
+ def wait_for_health(container, seconds):
926
+ deadline = time.monotonic() + seconds
927
+ health = "starting"
928
+ while time.monotonic() < deadline:
929
+ state, _, health = must([DOCKER, "inspect", "--format", "{{.State.Status}} {{if .State.Health}}{{.State.Health.Status}}{{end}}", container],
930
+ 30, "docker inspect").strip().partition(" ")
931
+ if state != "running":
932
+ return state
933
+ if health in ("healthy", "unhealthy"):
934
+ return health
935
+ time.sleep(5)
936
+ return f"still {health or 'without a health check'} after {seconds} s"
937
+
938
+
939
+ # Error names and codes only, never message text: enough to tell a Node or driver failure apart.
940
+ DIAGNOSTIC = re.compile(r"\b(?:ER_[A-Z_]{2,60}|ERR_[A-Z0-9_]{2,60}|[A-Z][A-Za-z]{1,40}Error|NODE_MODULE_VERSION [0-9]{1,4}"
941
+ r"|E(?:CONNREFUSED|NOTFOUND|ACCES|NOENT|ADDRINUSE|TIMEDOUT)|Cannot find module '[@A-Za-z0-9_./-]{1,120}')")
942
+
943
+
944
+ def error_names(container):
945
+ result = run([DOCKER, "logs", "--tail", "400", container], 60)
946
+ text = (result.stdout + result.stderr).decode("utf-8", "replace")
947
+ return sorted(set(DIAGNOSTIC.findall(text)))[:20]
948
+
949
+
950
+ def server_errors(container):
951
+ """MariaDB's and its entrypoint's own error lines, without repeats; they name a failed step, not the books."""
952
+ result = run([DOCKER, "logs", "--tail", "400", container], 60)
953
+ lines = []
954
+ for line in (result.stdout + result.stderr).decode("utf-8", "replace").splitlines():
955
+ found = re.search(r"\[ERROR\].*|ERROR [0-9]{4} \([0-9A-Z]{5}\).*", line)
956
+ text = found.group(0)[:200] if found else None
957
+ if text and text not in lines:
958
+ lines.append(text)
959
+ return lines[-6:]
960
+
961
+
962
+ def op_bigcapital_rehearse(args):
963
+ """Restores a backup into the current MariaDB on an internal network, upgrades it with the build's MariaDB,
964
+ runs the build's migrations, starts its server and Redis, and validates its proxy configuration; then
965
+ removes everything it created. A copy of the books never leaves the internal network."""
966
+ usage = "bigcapital-rehearse <stamp> <build>"
967
+ arity(args, 2, 2, usage)
968
+ build = need(BUILD, args[1], usage)
969
+ path = existing_record(args[0], usage)
970
+ source = path / BIGCAPITAL_DUMP
971
+ if source.is_symlink() or not source.is_file() or sha256_file(source) != (path / "SHA256SUMS").read_text(encoding="utf-8").split()[0]:
972
+ raise Failed("the backup's dump is missing or does not match its checksum")
973
+ record = path / f"rehearsal-{build}.json"
974
+ if record.exists():
975
+ raise Failed(f"{record.name} already exists; take a new backup to rehearse again")
976
+ current = must([DOCKER, "inspect", "--format", "{{.Image}}", BIGCAPITAL_DB], 30, "docker inspect").strip()
977
+ images = pull_build(build)
978
+ name = f"bigcapital-rehearsal-{build}"
979
+ db, migration, redis, server = (f"{name}-{part}" for part in ("db", "migration", "redis", "server"))
980
+ root_password, user_password, jwt_secret = (secrets.token_hex(24) for _ in range(3))
981
+ # The stack's database settings: the image's own MYSQL_DATABASE default names no database, and the
982
+ # upgrade's healthcheck-user step fails on it.
983
+ database = ["--network", name, "--network-alias", "mysql", "-v", f"{name}:/var/lib/mysql", "-e", f"MYSQL_ROOT_PASSWORD={root_password}",
984
+ "-e", f"MYSQL_DATABASE={BIGCAPITAL_SYSTEM_DB}", "-e", "MYSQL_USER=bigcapital", "-e", f"MYSQL_PASSWORD={user_password}"]
985
+ app_env = [part for key, value in {**BIGCAPITAL_SERVER_ENV, "DB_PASSWORD": user_password}.items() for part in ("-e", f"{key}={value}")]
986
+ checks = []
987
+ outcome = {"stamp": args[0], "build": build, "images": images, "current_mysql_image": current, "checks": checks}
988
+ step = "network and volume"
989
+ try:
990
+ must([DOCKER, "network", "create", "--internal", name], 60, "docker network create")
991
+ must([DOCKER, "volume", "create", name], 60, "docker volume create")
992
+ step = "restore into the current MariaDB"
993
+ must([DOCKER, "run", "-d", "--name", db, *database, current], 120, "docker run of the current MariaDB")
994
+ wait_for_server(db, 300)
995
+ restore_dump(db, source)
996
+ restored = fingerprint(db)
997
+ must([DOCKER, "stop", "-t", "300", db], 330, "docker stop of the current MariaDB")
998
+ must([DOCKER, "rm", db], 60, "docker rm")
999
+ step = "upgrade with the new MariaDB"
1000
+ must([DOCKER, "run", "-d", "--name", db, *database, "-e", "MARIADB_AUTO_UPGRADE=1", images["mysql"]], 120, "docker run of the new MariaDB")
1001
+ wait_for_server(db, 900)
1002
+ upgraded = fingerprint(db)
1003
+ version = maria(db, "SELECT VERSION()")[0][0]
1004
+ checks.append({"check": "upgrade kept every row and ledger", "ok": upgraded == restored,
1005
+ "observed": {"server_version": version, **differences(restored, upgraded)}})
1006
+ step = "migrations, server, Redis, and proxy"
1007
+ code = run([DOCKER, "run", "--name", migration, "--network", name, *app_env, images["migration"]], 1800).returncode
1008
+ checks.append({"check": "migrations exit 0", "ok": code == 0, "observed": code if code == 0 else {"exit": code, "errors": error_names(migration)}})
1009
+ migrated = fingerprint(db)
1010
+ changes = differences(upgraded, migrated)
1011
+ checks.append({"check": "migrations kept every ledger", "ok": not changes["ledgers"], "observed": changes})
1012
+ must([DOCKER, "run", "-d", "--name", redis, "--network", name, "--network-alias", "redis", images["redis"]], 120, "docker run of the new Redis")
1013
+ must([DOCKER, "run", "-d", "--name", server, "--network", name, *app_env, "-e", f"APP_JWT_SECRET={jwt_secret}", images["server"]],
1014
+ 120, "docker run of the new server")
1015
+ health = wait_for_health(server, 300)
1016
+ checks.append({"check": "server healthy", "ok": health == "healthy",
1017
+ "observed": health if health == "healthy" else {"health": health, "errors": error_names(server)}})
1018
+ pong = run([DOCKER, "exec", redis, "redis-cli", "ping"], 30).stdout.strip().decode("utf-8", "replace")
1019
+ checks.append({"check": "redis answers", "ok": pong == "PONG", "observed": pong[:40]})
1020
+ code = run([DOCKER, "run", "--rm", "--network", "none", images["proxy"], "envoy", "--mode", "validate", "-c", "/etc/envoy/envoy.yaml"],
1021
+ 120).returncode
1022
+ checks.append({"check": "proxy configuration valid", "ok": code == 0, "observed": code})
1023
+ outcome["fingerprint"] = summary(migrated)
1024
+ except Failed as error:
1025
+ checks.append({"check": "completed", "ok": False, "observed": {"step": step, "error": str(error), "database_errors": server_errors(db)}})
1026
+ finally:
1027
+ for container in (server, redis, migration, db):
1028
+ run([DOCKER, "rm", "-f", "-v", container], 120)
1029
+ run([DOCKER, "volume", "rm", name], 120)
1030
+ run([DOCKER, "network", "rm", name], 60)
1031
+ outcome["ok"] = all(c["ok"] for c in checks)
1032
+ write_json(record, outcome)
1033
+ emit(outcome)
1034
+ if not outcome["ok"]:
1035
+ raise Failed("the rehearsal failed; see the JSON above")
1036
+
1037
+
1038
+ def volume_of(container, destination):
1039
+ mounts = json.loads(must([DOCKER, "inspect", "--format", "{{json .Mounts}}", container], 30, "docker inspect"))
1040
+ for mount in mounts:
1041
+ if mount.get("Type") == "volume" and mount.get("Destination") == destination:
1042
+ return mount["Name"]
1043
+ raise Failed(f"{container} mounts no volume at {destination}")
1044
+
1045
+
1046
+ def mountpoint(volume):
1047
+ return Path(must([DOCKER, "volume", "inspect", "--format", "{{.Mountpoint}}", volume], 30, "docker volume inspect").strip())
1048
+
1049
+
1050
+ def tree(root):
1051
+ """Type, mode, owner, and a file's size and content for every entry: the copy must match the source exactly.
1052
+
1053
+ A directory's own size depends on its history, so it is left out.
1054
+ """
1055
+ top = root.lstat()
1056
+ entries = {".": (top.st_mode, top.st_uid, top.st_gid, None, None)}
1057
+ for current, dirs, files in os.walk(root):
1058
+ dirs.sort()
1059
+ for name in dirs + sorted(files):
1060
+ full = Path(current) / name
1061
+ info = full.lstat()
1062
+ if stat.S_ISREG(info.st_mode):
1063
+ size, content = info.st_size, sha256_file(full)
1064
+ else:
1065
+ size, content = None, os.readlink(full) if stat.S_ISLNK(info.st_mode) else None
1066
+ entries[str(full.relative_to(root))] = (info.st_mode, info.st_uid, info.st_gid, size, content)
1067
+ return entries
1068
+
1069
+
1070
+ def tree_bytes(entries):
1071
+ return sum(size for _, _, _, size, _ in entries.values() if size is not None)
1072
+
1073
+
1074
+ def copy_volume(source, target):
1075
+ """cp -a into the new volume, its top directory's owner and mode, then a full comparison."""
1076
+ must(["/bin/cp", "-a", "--", f"{source}/.", f"{target}/"], 1800, "cp -a")
1077
+ top = source.lstat()
1078
+ os.chown(target, top.st_uid, top.st_gid)
1079
+ os.chmod(target, stat.S_IMODE(top.st_mode))
1080
+ expected = tree(source)
1081
+ if tree(target) != expected:
1082
+ raise Failed(f"the copy of {source} differs from its source")
1083
+ return {"entries": len(expected), "bytes": tree_bytes(expected)}
1084
+
1085
+
1086
+ def op_bigcapital_switch(args):
1087
+ """Owner-approved. With a passed rehearsal of the build the astrofoundry tags now name, stops the app,
1088
+ takes the quiescent dump, saves Redis, stops both, and copies their volumes to new ones. The stack stays
1089
+ stopped for the GitOps change that points it at the copies; the old volumes stay as they were for a
1090
+ rollback. A failure before the end starts the stopped containers again."""
1091
+ usage = "bigcapital-switch <stamp> <build>"
1092
+ arity(args, 2, 2, usage)
1093
+ build = need(BUILD, args[1], usage)
1094
+ path = existing_record(args[0], usage)
1095
+ record = path / f"rehearsal-{build}.json"
1096
+ if record.is_symlink() or not record.is_file() or json.loads(record.read_text(encoding="utf-8")).get("ok") is not True:
1097
+ raise Failed(f"no passed rehearsal of {build} on backup {args[0]}; nothing was stopped")
1098
+ rehearsed = json.loads(record.read_text(encoding="utf-8"))["images"]
1099
+ built = pull_build(build)
1100
+ if built != rehearsed:
1101
+ raise Failed(f"the images of {build} changed since the rehearsal; nothing was stopped")
1102
+ if pull_build("astrofoundry") != built:
1103
+ raise Failed(f"the astrofoundry tags do not name build {build}; nothing was stopped")
1104
+ for container in (BIGCAPITAL_DB, BIGCAPITAL_REDIS, *BIGCAPITAL_APP):
1105
+ if not container_running(container):
1106
+ raise Failed(f"{container} is not running; nothing was stopped")
1107
+ sources = {"mysql": volume_of(BIGCAPITAL_DB, "/var/lib/mysql"), "redis": volume_of(BIGCAPITAL_REDIS, "/data")}
1108
+ needed = sum(tree_bytes(tree(mountpoint(volume))) for volume in sources.values())
1109
+ if shutil.disk_usage(mountpoint(sources["mysql"])).free < 2 * needed:
1110
+ raise Failed(f"less than twice the volumes' {needed} bytes free; nothing was stopped")
1111
+ stamp, switch = new_record()
1112
+ targets = {kind: f"bigcapital_{kind}_{stamp}" for kind in sources}
1113
+ stopped = []
1114
+ try:
1115
+ for container in BIGCAPITAL_APP:
1116
+ must([DOCKER, "stop", "-t", "60", container], 90, f"docker stop {container}")
1117
+ stopped.append(container)
1118
+ backup = bigcapital_record(stamp, switch, quiescent=True)
1119
+ if run([DOCKER, "exec", BIGCAPITAL_REDIS, "redis-cli", "SAVE"], 300).stdout.strip() != b"OK":
1120
+ raise Failed("redis-cli SAVE did not answer OK")
1121
+ for container in (BIGCAPITAL_DB, BIGCAPITAL_REDIS):
1122
+ must([DOCKER, "stop", "-t", "300", container], 330, f"docker stop {container}")
1123
+ stopped.append(container)
1124
+ copies = {}
1125
+ for kind, volume in sources.items():
1126
+ must([DOCKER, "volume", "create", targets[kind]], 60, "docker volume create")
1127
+ copies[kind] = {"from": volume, "to": targets[kind], **copy_volume(mountpoint(volume), mountpoint(targets[kind]))}
1128
+ except Exception as error:
1129
+ for container in reversed(stopped):
1130
+ run([DOCKER, "start", container], 120)
1131
+ detail = str(error) if isinstance(error, Failed) else type(error).__name__
1132
+ raise Failed(f"{detail}; the stopped containers were started again and the old volumes are unchanged") from None
1133
+ outcome = {"stamp": stamp, "path": str(switch), "build": build, "rehearsal": args[0], "images": built, "volumes": copies,
1134
+ "backup": {k: backup[k] for k in ("file", "sha256", "server_version", "fingerprint")}, "stopped": stopped}
1135
+ write_json(switch / "switch.json", outcome)
1136
+ emit(outcome)
1137
+
1138
+
1139
+ def op_bigcapital_verify(args):
1140
+ """After the GitOps change: the copies are mounted, the build runs, and the books match the switch's dump."""
1141
+ usage = "bigcapital-verify <stamp>"
1142
+ arity(args, 1, 1, usage)
1143
+ path = existing_record(args[0], usage)
1144
+ if not (path / "switch.json").is_file():
1145
+ raise Failed(f"{args[0]} is not a completed switch")
1146
+ switch = json.loads((path / "switch.json").read_text(encoding="utf-8"))
1147
+ before = json.loads((path / "fingerprint.json").read_text(encoding="utf-8"))
1148
+ checks = []
1149
+ for kind, container, destination in (("mysql", BIGCAPITAL_DB, "/var/lib/mysql"), ("redis", BIGCAPITAL_REDIS, "/data")):
1150
+ observed = volume_of(container, destination)
1151
+ checks.append({"check": f"{kind} mounts its copy", "ok": observed == switch["volumes"][kind]["to"], "observed": observed})
1152
+ for service in ("mysql", "redis", "server", "webapp", "proxy", "migration"):
1153
+ container = f"{BIGCAPITAL}-{'database_migration' if service == 'migration' else service}-1"
1154
+ observed = run([DOCKER, "inspect", "--format", "{{.Image}}", container], 30).stdout.strip().decode()
1155
+ checks.append({"check": f"{service} runs the build", "ok": observed == switch["images"][service], "observed": observed[:19]})
1156
+ exit_code = run([DOCKER, "inspect", "--format", "{{.State.ExitCode}}", f"{BIGCAPITAL}-database_migration-1"], 30).stdout.strip().decode()
1157
+ checks.append({"check": "migrations exit 0", "ok": exit_code == "0", "observed": exit_code})
1158
+ health = wait_for_health("bigcapital-server-1", 300)
1159
+ checks.append({"check": "server healthy", "ok": health == "healthy", "observed": health})
1160
+ after = fingerprint(BIGCAPITAL_DB)
1161
+ changes = differences(before, after)
1162
+ checks.append({"check": "ledgers match the switch's dump", "ok": not changes["ledgers"], "observed": changes["ledgers"]})
1163
+ checks.append({"check": "row counts match the switch's dump", "ok": not changes["tables"], "observed": changes["tables"][:20]})
1164
+ emit({"ok": all(c["ok"] for c in checks), "server_version": maria(BIGCAPITAL_DB, "SELECT VERSION()")[0][0],
1165
+ "fingerprint": summary(after), "checks": checks})
1166
+
1167
+
719
1168
  COMMANDS = {
720
1169
  "container-memory": op_container_memory,
721
1170
  "container-stats": op_container_stats,
@@ -748,6 +1197,10 @@ COMMANDS = {
748
1197
  "db-dump": op_db_dump,
749
1198
  "db-restore": op_db_restore,
750
1199
  "supabase-backup": op_supabase_backup,
1200
+ "bigcapital-backup": op_bigcapital_backup,
1201
+ "bigcapital-rehearse": op_bigcapital_rehearse,
1202
+ "bigcapital-switch": op_bigcapital_switch,
1203
+ "bigcapital-verify": op_bigcapital_verify,
751
1204
  }
752
1205
 
753
1206
 
@@ -28,6 +28,8 @@ describe("Arcane host commands", () => {
28
28
  expect(buildHostRemote(["db-restore", "mealie", "20261019T120000Z"]).name).toBe("db-restore");
29
29
  expect(buildHostRemote(["mealie-api"]).spec.stdinSecret).toBe("MEALIE_API_KEY");
30
30
  expect(buildHostRemote(["supabase-backup"]).remote).toBe("supabase-backup");
31
+ expect(buildHostRemote(["bigcapital-rehearse", "20261104T080000Z", "a87df4f"]).remote).toBe("bigcapital-rehearse 20261104T080000Z a87df4f");
32
+ expect(buildHostRemote(["bigcapital-verify", "20261104T090000Z"]).remote).toBe("bigcapital-verify 20261104T090000Z");
31
33
  const bad = [
32
34
  [],
33
35
  ["rm"],
@@ -43,6 +45,10 @@ describe("Arcane host commands", () => {
43
45
  ["db-dump", "supabase"],
44
46
  ["db-restore", "mealie", "latest"],
45
47
  ["supabase-backup", "_supabase"],
48
+ ["bigcapital-backup", "now"],
49
+ ["bigcapital-rehearse", "20261104T080000Z", "astrofoundry"],
50
+ ["bigcapital-switch", "latest", "a87df4f"],
51
+ ["bigcapital-verify", "20261104T090000Z", "a87df4f"],
46
52
  ["container-stats", ...Array.from({ length: 9 }, (_, i) => `c${i}`)],
47
53
  ["manager-start"],
48
54
  ["manager-rollback", "20261019T120000Z"],
@@ -47,6 +47,10 @@ export const HOST_COMMANDS: Readonly<Record<string, HostCommand>> = {
47
47
  "db-dump": { args: [1, 1], help: "db-dump mealie|paperless pg_dump into /var/backups/spc-arcane/<stack>/<UTC> (gzip, checked, sha256)", timeoutMs: 60 * MINUTE },
48
48
  "db-restore": { args: [2, 2], help: "db-restore <stack> <stamp> fill an EMPTY database from one of these dumps, the app stopped meanwhile (owner approval)", timeoutMs: 60 * MINUTE },
49
49
  "supabase-backup": { args: [0, 0], help: "supabase-backup both Supabase databases, roles, and db-config (pgsodium key) into /var/backups/supabase-upgrades/<UTC>, checked", timeoutMs: 90 * MINUTE },
50
+ "bigcapital-backup": { args: [0, 0], help: "bigcapital-backup online dump of the Bigcapital databases, row counts, ledger totals, image digests into /var/backups/bigcapital-upgrades/<UTC>", timeoutMs: 60 * MINUTE },
51
+ "bigcapital-rehearse": { args: [2, 2], help: "bigcapital-rehearse <stamp> <build> restore that backup on an internal network, upgrade it with the build's MariaDB, run its migrations, start its server", timeoutMs: 120 * MINUTE },
52
+ "bigcapital-switch": { args: [2, 2], help: "bigcapital-switch <stamp> <build> stop Bigcapital, quiescent dump, copy its volumes to new ones for the GitOps switch (owner approval)", timeoutMs: 90 * MINUTE },
53
+ "bigcapital-verify": { args: [1, 1], help: "bigcapital-verify <stamp> after the switch: copied volumes mounted, the build running, books matching the switch's dump", timeoutMs: 15 * MINUTE },
50
54
  "manager-state": { args: [0, 0], help: "manager-state", timeoutMs: MINUTE, internal: true },
51
55
  "manager-prepare": { args: [1, 1], help: "manager-prepare <stamp>", timeoutMs: 2 * MINUTE, internal: true },
52
56
  "manager-pull": { args: [1, 1], help: "manager-pull <digest>", timeoutMs: 16 * MINUTE, internal: true },
@@ -59,6 +63,8 @@ export const HOST_COMMANDS: Readonly<Record<string, HostCommand>> = {
59
63
  const NAME = /^[A-Za-z0-9][A-Za-z0-9_.-]{0,127}$/;
60
64
  const UTC = /^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}Z$/;
61
65
  export const STAMP = /^\d{8}T\d{6}Z$/;
66
+ /** A fork build: the seven-character commit tag the bigcapital CI publishes. */
67
+ const BUILD = /^[0-9a-f]{7}$/;
62
68
  const IMDB = /^tt\d{1,12}$/;
63
69
  const UUID = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/;
64
70
  const OBJECT_PATH = /^[A-Za-z0-9_][A-Za-z0-9_./-]{0,255}$/;
@@ -120,6 +126,14 @@ export function buildHostRemote(args: string[], allowInternal = false): { remote
120
126
  check((DB_STACKS as readonly string[]).includes(rest[0]), `stack: ${DB_STACKS.join(", ")}`);
121
127
  check(STAMP.test(rest[1]), "stamp: YYYYMMDDTHHMMSSZ from db-list");
122
128
  break;
129
+ case "bigcapital-rehearse":
130
+ case "bigcapital-switch":
131
+ check(STAMP.test(rest[0]), "stamp: YYYYMMDDTHHMMSSZ from bigcapital-backup");
132
+ check(BUILD.test(rest[1]), "build: the seven-character commit tag of the fork's images");
133
+ break;
134
+ case "bigcapital-verify":
135
+ check(STAMP.test(rest[0]), "stamp: YYYYMMDDTHHMMSSZ from bigcapital-switch");
136
+ break;
123
137
  default:
124
138
  break;
125
139
  }