@astrofoundry/pi-astro 0.40.48 → 0.40.49
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/agents/arcane.md +1 -1
- package/extensions/specialist-gate/approvals.test.ts +2 -0
- package/extensions/specialist-gate/index.ts +2 -2
- package/package.json +1 -1
- package/skills/arcane/SKILL.md +6 -0
- package/specialists/AGENTS.md +1 -1
- package/specialists/entry/remote/spc-arcane-op.py +435 -0
- package/specialists/lib/arcane-host.test.ts +6 -0
- package/specialists/lib/arcane-host.ts +14 -0
package/agents/arcane.md
CHANGED
|
@@ -20,7 +20,7 @@ Rules:
|
|
|
20
20
|
- Operational actions (restart, logs, stats, redeploy, image pull, prune) go straight through the CLI.
|
|
21
21
|
- Confirm destructive operations (`down`, `delete`, `prune`) are explicitly requested in the task before running them.
|
|
22
22
|
- Updating Arcane itself is `manager-upgrade <version>` (owner approval), which runs the whole documented procedure and rolls back on failed checks; the owner's Upgrade button on the Arcane Daily Report runs the same. Never run `system upgrade`, `projects upgrade arcane`, or a manual sync of `arcane`, and never hand host commands to the caller.
|
|
23
|
-
- Host evidence (memory, logs, OOM, capacity, storage, audit) comes from `host <command>`, never from guesses; `host db-restore` only fills an empty database and needs approval.
|
|
23
|
+
- Host evidence (memory, logs, OOM, capacity, storage, audit) comes from `host <command>`, never from guesses; `host db-restore` only fills an empty database and needs approval; `host bigcapital-switch` stops Bigcapital and needs approval.
|
|
24
24
|
- For update questions run `update-report` and answer with its table verbatim in a code block; never answer from `projects updates`, which is stored and paginated.
|
|
25
25
|
- Image updates that `update-report` offers go through `stack-update <stack>`, one stack at a time and only when asked; it edits, commits, redeploys, checks, and rolls back on its own. It refuses held stacks (`update-holds.json`) and `arcane`.
|
|
26
26
|
- Any other image change to one service the caller asks for (a tag switch, a pin, a digest) is a definition change: change only that `image:` line through the GitOps path (`git write`, `git diff`, `git commit`, `git push`, then the stack's `gitops sync`), as the skill describes. It is not a reason to refuse.
|
|
@@ -45,6 +45,8 @@ describe("approvals", () => {
|
|
|
45
45
|
expect(isRisky("arcane", ["host", "db-restore", "mealie", "20261019T120000Z"])).toBe(true);
|
|
46
46
|
expect(isRisky("arcane", ["host", "radarr-movie", "tt0120737"])).toBe(true);
|
|
47
47
|
expect(isRisky("arcane", ["host", "db-dump", "mealie"])).toBe(false);
|
|
48
|
+
expect(isRisky("arcane", ["host", "bigcapital-switch", "20261104T080000Z", "a87df4f"])).toBe(true);
|
|
49
|
+
expect(isRisky("arcane", ["host", "bigcapital-rehearse", "20261104T080000Z", "a87df4f"])).toBe(false);
|
|
48
50
|
expect(isRisky("arcane", ["host", "container-logs", "sonarr", "2026-09-06T12:00:00Z", "2026-09-06T12:15:00Z"])).toBe(false);
|
|
49
51
|
expect(isRisky("edge", ["vps", "reboot", "--confirm", "--help"])).toBe(true);
|
|
50
52
|
expect(isRisky("dns", ["technitium", "primary", "zone-delete", "example.com", "--help"])).toBe(true);
|
|
@@ -27,9 +27,9 @@ export const RISKY: Readonly<Record<string, readonly RegExp[]>> = {
|
|
|
27
27
|
/^(images|volumes|networks)\s+(prune|remove|rm|delete)\b/,
|
|
28
28
|
/^system\s+(prune|upgrade)\b/,
|
|
29
29
|
/^gitops\s+delete\b/,
|
|
30
|
-
// Arcane VM: upgrading or rolling back the manager, filling a database, and reading Radarr's key from its host config.
|
|
30
|
+
// Arcane VM: upgrading or rolling back the manager, filling a database, stopping Bigcapital for its switch, and reading Radarr's key from its host config.
|
|
31
31
|
/^manager-(upgrade|rollback)\b/,
|
|
32
|
-
/^host\s+(db-restore|radarr-movie|radarr-interface)\b/,
|
|
32
|
+
/^host\s+(db-restore|bigcapital-switch|radarr-movie|radarr-interface)\b/,
|
|
33
33
|
],
|
|
34
34
|
identity: [/^zitadel\s+(DELETE|POST|PUT|PATCH)\b/, /^dmz\s+(deploy-config|pomerium-restart|pomerium-upgrade|render|lego-renew)\b/],
|
|
35
35
|
// Every UniFi write, on either console.
|
package/package.json
CHANGED
package/skills/arcane/SKILL.md
CHANGED
|
@@ -88,6 +88,12 @@ The usual trigger is the owner's **Upgrade Arcane** button under the Arcane Dail
|
|
|
88
88
|
- `db-list mealie|paperless`, `db-dump mealie|paperless` (gzip pg_dump into `/var/backups/spc-arcane/<stack>/<UTC>`, checked, with row counts), `db-restore <stack> <stamp>` (owner approval): fills an **empty** database from one of those dumps, the app container stopped meanwhile. It refuses a database that holds tables; emptying one is an operator decision. Dumps are never deleted by the tool; `db-list` names those beyond the last three in `over_keep`.
|
|
89
89
|
- `supabase-backup`: the backup `supabase/AGENTS.md` asks for before a Supabase change. It writes custom-format dumps of `postgres` and `_supabase`, `globals.sql`, and `db-config.tar.gz` (the pgsodium root key) into `/var/backups/supabase-upgrades/<UTC>`, checks each dump with pg_restore's listing and full decoding, and prints the manifest with the image's repo digests (record them as the rollback digest). It refuses when the disk has less free space than the databases. Restoring stays an operator step.
|
|
90
90
|
|
|
91
|
+
- Bigcapital base-image upgrades (`bigcapital/README.md` "Base-image upgrades"), all under `/var/backups/bigcapital-upgrades/<UTC>`:
|
|
92
|
+
- `bigcapital-backup`: an online, consistent dump of the system and tenant databases, with exact row counts, each tenant's ledger totals (kept on the host, never printed), and the running image digests.
|
|
93
|
+
- `bigcapital-rehearse <stamp> <build>`: restores that dump into the current MariaDB on an internal Docker network, upgrades it with the build's MariaDB (`MARIADB_AUTO_UPGRADE`), runs the build's migrations, starts its Redis and server, and validates its Envoy configuration. It compares rows and ledgers at each step, removes everything it created, and prints error names only.
|
|
94
|
+
- `bigcapital-switch <stamp> <build>` (owner approval): needs a passed rehearsal of that build and the `astrofoundry` tags pointing at it. It stops the app, takes the quiescent dump, saves Redis, stops both, and copies their volumes to `bigcapital_mysql_<UTC>` and `bigcapital_redis_<UTC>`. The stack stays stopped for the GitOps commit that mounts the copies; a failure starts the stopped containers again. The old volumes stay untouched for a rollback.
|
|
95
|
+
- `bigcapital-verify <stamp>`: after that commit's redeploy, checks the mounted copies, the running build, the migration exit code, server health, and rows and ledgers against the switch's dump.
|
|
96
|
+
|
|
91
97
|
Public routes (Supabase, Grimoire, Mealie URLs) are `astro.edge`'s `route-probe`, run from Cortex.
|
|
92
98
|
|
|
93
99
|
`["stack-rollback", "<stack>", "<commit>"]` undoes one earlier `stack-update` commit of that stack's Compose file (subject `<stack>: ...`, only that file, on the branch): revert, push, start the sync, redeploy, recreate skipped containers, check. The Discord bridge runs it when a stack fails its check minutes after an automatic update; run it yourself only when the caller asks for that rollback.
|
package/specialists/AGENTS.md
CHANGED
|
@@ -59,7 +59,7 @@ Wrappers ship as TypeScript and run under Node 24 type stripping: erasable synta
|
|
|
59
59
|
- `main()` waits for stdout and stderr to flush before `process.exit`: on a pipe, exiting right after a write drops everything past the 64 KiB buffer, which cut large answers such as `projects list --all` until 2026-09-25 (`lib/flush.test.ts`).
|
|
60
60
|
- `arcane update-report [--all]` is `lib/image-updates.ts` over `lib/registry.ts`: a fresh `images updates check --all`, then `projects list --all` for the stack and service of each image reference, then the registries' tag lists and manifest digests (anonymous bearer tokens from each registry's challenge; digests by HEAD, which Docker Hub does not count as a pull; lscr.io is read from ghcr.io). A floating or major-series tag is resolved to the release tag whose digest matches the running and the latest build, searching the newest tags of the three commonest release shapes; a fixed tag is compared with the newest tag of the same shape; a digest pin with the newest release. Checks older than one minute before the run count as stale and show as unknown. Private images (anonymous access refused with 401 or 403) are excluded from rows, counts, and table, listed in `privateImages`, and counted in the summary line. The table is fixed-width text for a code block, because Discord renders no Markdown tables. The answer adds `holds` from `update-holds.json` at origin's branch head (`git show`, no working-tree read), or `holdsError` when that file is malformed.
|
|
61
61
|
- `arcane stack-update <stack> [--confirm]` (`lib/stack-update.ts` for the pure parts) is deterministic: refuses `arcane`, held stacks (the checkout's `update-holds.json` after `pull --ff-only`; a malformed file is an error), a dirty or ahead checkout, stopped stacks, and an update across a major version without `--confirm` (which the gate turns into an owner approval). Pending rows come from a fresh per-image check resolved like `update-report`. `edit tag`/`edit digest` rows rewrite `image:` lines that name exactly the old reference (`replaceImageRef`, quoted or not, trailing comment kept) in the sync's `composePath`, then commit, push, and wait for `gitops list` to report success at that commit (20 min, polled every 30 s). Then `projects redeploy <stack> --yes` (35 min), container checks through `containers list --all` and `containers get` (`composeInfo.projectName`, health; 5 min, polled every 15 s; a job that exited with code 0 counts as done whatever health its image's inherited check left, as bigcapital's migration did on 2026-10-28), and a fresh check that each service is current. A failure after a commit reverts, pushes, waits for the sync, redeploys, and checks again (`rolled back`); a pull-only failure stays `failed`. Every outcome prints JSON with `status` and exits 0. Worst case about 125 minutes, so the gate's `timeoutMs` for arcane must exceed it (8,100,000). It holds the `arcane-gitops` lock, so a second checkout call waits up to 2 minutes and then fails as busy; the bridge also runs one Arcane update at a time.
|
|
62
|
-
- `arcane host <command>` runs over the forced-command key of user `spc-arcane` on VM 100. `HOST_COMMANDS` in `lib/arcane-host.ts` must list exactly the operations of `entry/remote/spc-arcane-op.py` (a test compares them); the tracked copies live in the arcane repository's `host/` (`spc-arcane-host-entry`, `spc-arcane-op`, `spc-arcane-sudoers`, `spc-arcane-install.sh`) with the probes in `host/spc-arcane-probes/`. The entry only splits words (letters, digits, space, `._:/+-`) and hands them to `sudo -n /usr/local/sbin/spc-arcane-op`, the one command its sudoers line allows; the op checks the name and every argument and runs the reviewed arcane-ops probes, which print only allowlisted fields. Manager steps (`manager-*`) are internal to `manager-upgrade`/`manager-rollback` and refused through `host`. Nothing on the host deletes data: a manager rollback restores the image and Compose file, not the data volume (R3 stays manual), `db-restore` only fills an empty database, and dumps are never pruned by the tool. `supabase-backup` writes the pre-change backup of `supabase/AGENTS.md` (both databases with `pg_dump -Fc`, each listed and fully decoded by `pg_restore`, `pg_dumpall --globals-only`, and the `supabase-db-config` volume, which must hold `pgsodium_root.key`) into `/var/backups/supabase-upgrades/<UTC>`, the folder of the earlier manual backups. `host-audit` compares the host's hashes with `host/<name>` of the `arcane-gitops` checkout after a pull.
|
|
62
|
+
- `arcane host <command>` runs over the forced-command key of user `spc-arcane` on VM 100. `HOST_COMMANDS` in `lib/arcane-host.ts` must list exactly the operations of `entry/remote/spc-arcane-op.py` (a test compares them); the tracked copies live in the arcane repository's `host/` (`spc-arcane-host-entry`, `spc-arcane-op`, `spc-arcane-sudoers`, `spc-arcane-install.sh`) with the probes in `host/spc-arcane-probes/`. The entry only splits words (letters, digits, space, `._:/+-`) and hands them to `sudo -n /usr/local/sbin/spc-arcane-op`, the one command its sudoers line allows; the op checks the name and every argument and runs the reviewed arcane-ops probes, which print only allowlisted fields. Manager steps (`manager-*`) are internal to `manager-upgrade`/`manager-rollback` and refused through `host`. Nothing on the host deletes data: a manager rollback restores the image and Compose file, not the data volume (R3 stays manual), `db-restore` only fills an empty database, and dumps are never pruned by the tool. `supabase-backup` writes the pre-change backup of `supabase/AGENTS.md` (both databases with `pg_dump -Fc`, each listed and fully decoded by `pg_restore`, `pg_dumpall --globals-only`, and the `supabase-db-config` volume, which must hold `pgsodium_root.key`) into `/var/backups/supabase-upgrades/<UTC>`, the folder of the earlier manual backups. `host-audit` compares the host's hashes with `host/<name>` of the `arcane-gitops` checkout after a pull. The `bigcapital-*` commands run the base-image upgrade of `bigcapital/README.md`: an online backup, a rehearsal on an internal Docker network (restore into the current MariaDB, upgrade with the build's, migrations, server health, Envoy validation; everything it created removed afterwards), the owner-approved switch (stop, quiescent dump, Redis `SAVE`, volume copies compared entry by entry, the stack left stopped for the GitOps commit, the old volumes untouched), and a verify against the switch's row counts and ledger totals.
|
|
63
63
|
- `arcane manager-upgrade <version>` (`lib/arcane-manager.ts`) follows arcane/AGENTS.md "Manager upgrades": the official image's revision label must equal the release tag's commit (GitHub API), `manager-prepare` records `compose.before.yaml` and every container before the pin commit, the pin goes through the `arcane-gitops` checkout (`git` is exported from `lib/repo.ts` for this), the wait polls `gitops list` and the deployed Compose hash every 30 s for up to 30 minutes, then pull, cold backup, `up -d --no-deps --pull never`, and checks. Any failure before the restart reverts the pin (`stopped`); a failed check after it reverts the pin and runs `manager-rollback` (`rolled back`). The answer always exits 0 with `status`, so the bridge and the agent both see it.
|
|
64
64
|
- `media` runs trendarr unchanged: `trendarrCli` in `media.json` is its `dist/cli.js`, started with `process.execPath` because sudo's PATH has no node. Each call gets `HOME` set to a fresh `0700` directory under `~/.specialists/work/media/` that holds only `.trendarr/config.json` (`0600`, trendarr's own schema, built from the note fields and `media.json`) and is removed in `finally`; trendarr reads no environment variables and has no config flag, and Node's `os.homedir()` follows `$HOME` on POSIX. Every call passes `--json`; `init` is refused. An add prints trendarr's per-ID `results` and exits 0 even when trendarr exits 1 for a failed ID, because the gate shows only stderr on failure; other failures carry trendarr's stderr without its `warning:` lines, which go into the JSON as `warnings`. The bridge's slash commands and Add buttons call the same wrapper through the entry point (`extensions/astro-discord/media.ts`).
|
|
65
65
|
- `arcane release-notes <stack>[/<service>]` is `lib/releases.ts`: a single-image check and `resolveRef` give the running and latest versions, `HttpRegistry.labels` reads `org.opencontainers.image.source` from the latest image (index to linux/amd64 manifest to config blob), and the anonymous GitHub releases API (60 requests an hour, up to 3 pages of 100) gives the releases in between, oldest first. `condenseNotes` keeps each release under 12,000 characters by section priority; `capTotal` drops the oldest releases past 40,000 characters in total.
|
|
@@ -16,7 +16,9 @@ import importlib.util
|
|
|
16
16
|
import json
|
|
17
17
|
import os
|
|
18
18
|
import re
|
|
19
|
+
import secrets
|
|
19
20
|
import shutil
|
|
21
|
+
import stat
|
|
20
22
|
import subprocess
|
|
21
23
|
import sys
|
|
22
24
|
import tarfile
|
|
@@ -56,6 +58,33 @@ SUPABASE_CONFIG = Path("/var/lib/docker/volumes/supabase-db-config/_data")
|
|
|
56
58
|
SUPABASE_KEY = "./pgsodium_root.key"
|
|
57
59
|
SUPABASE_BACKUPS = Path("/var/backups/supabase-upgrades")
|
|
58
60
|
|
|
61
|
+
# Bigcapital base-image upgrades (bigcapital/README.md "Update the app"): the
|
|
62
|
+
# Compose containers, the fork's images, and the earlier manual backups' folder.
|
|
63
|
+
BIGCAPITAL = "bigcapital"
|
|
64
|
+
BIGCAPITAL_DB = "bigcapital-mysql-1"
|
|
65
|
+
BIGCAPITAL_REDIS = "bigcapital-redis-1"
|
|
66
|
+
# Stopped first, started last: nothing writes while the database is dumped and copied.
|
|
67
|
+
BIGCAPITAL_APP = ("bigcapital-proxy-1", "bigcapital-webapp-1", "bigcapital-server-1")
|
|
68
|
+
BIGCAPITAL_IMAGE = "ghcr.io/astrofoundry/bigcapital-{}:{}"
|
|
69
|
+
BIGCAPITAL_SERVICES = ("mysql", "redis", "server", "webapp", "proxy", "migration")
|
|
70
|
+
BIGCAPITAL_BACKUPS = Path("/var/backups/bigcapital-upgrades")
|
|
71
|
+
BIGCAPITAL_DUMP = "bigcapital-databases.sql.gz"
|
|
72
|
+
BIGCAPITAL_SYSTEM_DB = "bigcapital_system"
|
|
73
|
+
BIGCAPITAL_TENANT_PREFIX = "bigcapital_tenant_"
|
|
74
|
+
# The server's non-secret settings from bigcapital/compose.yaml, without mail and
|
|
75
|
+
# public URLs; the integrations stay explicit and empty as there.
|
|
76
|
+
BIGCAPITAL_SERVER_ENV = {
|
|
77
|
+
"DB_HOST": "mysql", "DB_USER": "bigcapital", "DB_CHARSET": "utf8", "SYSTEM_DB_NAME": BIGCAPITAL_SYSTEM_DB,
|
|
78
|
+
"TENANT_DB_NAME_PERFIX": BIGCAPITAL_TENANT_PREFIX, "REDIS_HOST": "redis", "REDIS_PORT": "6379", "QUEUE_HOST": "redis",
|
|
79
|
+
"QUEUE_PORT": "6379", "SIGNUP_DISABLED": "false", "SIGNUP_EMAIL_CONFIRMATION": "false", "EXCHANGE_RATE_SERVICE": "open-exchange-rate", "CLICKHOUSE_ENABLED": "false",
|
|
80
|
+
"PLAID_ENV": "sandbox", "S3_REGION": "US", **{name: "" for name in (
|
|
81
|
+
"SIGNUP_ALLOWED_DOMAINS", "SIGNUP_ALLOWED_EMAILS", "OPEN_EXCHANGE_RATE_APP_ID", "BANK_FEED_ENABLED", "PLAID_CLIENT_ID", "PLAID_SECRET", "PLAID_LINK_WEBHOOK",
|
|
82
|
+
"LEMONSQUEEZY_API_KEY", "LEMONSQUEEZY_STORE_ID", "LEMONSQUEEZY_WEBHOOK_SECRET", "HOSTED_ON_BIGCAPITAL_CLOUD",
|
|
83
|
+
"S3_ACCESS_KEY_ID", "S3_SECRET_ACCESS_KEY", "S3_ENDPOINT", "S3_BUCKET", "STRIPE_PAYMENT_SECRET_KEY",
|
|
84
|
+
"STRIPE_PAYMENT_PUBLISHABLE_KEY", "STRIPE_PAYMENT_CLIENT_ID", "STRIPE_PAYMENT_WEBHOOKS_SECRET", "STRIPE_PAYMENT_REDIRECT_URL")},
|
|
85
|
+
}
|
|
86
|
+
SQL_CLIENT = 'exec "$(command -v mariadb || command -v mysql)" -uroot -p"$MYSQL_ROOT_PASSWORD"'
|
|
87
|
+
|
|
59
88
|
# Tracked host files the audit compares with the repository's host/ folder.
|
|
60
89
|
HOST_FILES = (
|
|
61
90
|
("arcane-backup.sh", "/usr/local/bin/arcane-backup.sh"),
|
|
@@ -102,6 +131,8 @@ VERSION = re.compile(r"^v?[0-9]{1,4}\.[0-9]{1,4}\.[0-9]{1,4}$")
|
|
|
102
131
|
IMDB = re.compile(r"^tt[0-9]{1,12}$")
|
|
103
132
|
UUID = re.compile(r"^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$")
|
|
104
133
|
INTEGER = re.compile(r"^[0-9]{1,9}$")
|
|
134
|
+
BUILD = re.compile(r"^[0-9a-f]{7}$")
|
|
135
|
+
SQL_NAME = re.compile(r"^[A-Za-z0-9_]{1,64}$")
|
|
105
136
|
OBJECT_PATH = re.compile(r"^[A-Za-z0-9_][A-Za-z0-9_./-]{0,255}$")
|
|
106
137
|
|
|
107
138
|
|
|
@@ -716,6 +747,406 @@ def op_supabase_backup(args):
|
|
|
716
747
|
emit(info)
|
|
717
748
|
|
|
718
749
|
|
|
750
|
+
# ---------------------------------------------------------------- bigcapital base upgrades
|
|
751
|
+
|
|
752
|
+
|
|
753
|
+
def maria(container, statement, timeout=600):
|
|
754
|
+
"""One statement through the container's own client as root; rows come back as lists of fields."""
|
|
755
|
+
result = run([DOCKER, "exec", container, "sh", "-c", SQL_CLIENT + ' -N -B -e "$1"', "sh", statement], timeout)
|
|
756
|
+
if result.returncode != 0:
|
|
757
|
+
raise Failed(f"SQL in {container} failed (exit {result.returncode}); its output was withheld")
|
|
758
|
+
return [line.split("\t") for line in result.stdout.decode("utf-8", "replace").splitlines()]
|
|
759
|
+
|
|
760
|
+
|
|
761
|
+
def wait_for_server(container, seconds):
|
|
762
|
+
"""A TCP ping: the entrypoint's temporary init and upgrade servers listen only on the socket, so they never count."""
|
|
763
|
+
ping = 'exec "$(command -v mariadb-admin || command -v mysqladmin)" ping -h127.0.0.1 --protocol=tcp'
|
|
764
|
+
deadline = time.monotonic() + seconds
|
|
765
|
+
while time.monotonic() < deadline:
|
|
766
|
+
if run([DOCKER, "exec", container, "sh", "-c", ping], 30).returncode == 0:
|
|
767
|
+
return
|
|
768
|
+
if not container_running(container):
|
|
769
|
+
raise Failed(f"{container} stopped before its server accepted connections")
|
|
770
|
+
time.sleep(3)
|
|
771
|
+
raise Failed(f"{container} did not accept connections within {seconds} s")
|
|
772
|
+
|
|
773
|
+
|
|
774
|
+
def bigcapital_schemas(container):
|
|
775
|
+
pattern = BIGCAPITAL_TENANT_PREFIX.replace("_", "\\_") + "%"
|
|
776
|
+
rows = maria(container, f"SELECT schema_name FROM information_schema.schemata WHERE schema_name = '{BIGCAPITAL_SYSTEM_DB}' "
|
|
777
|
+
f"OR schema_name LIKE '{pattern}' ORDER BY schema_name")
|
|
778
|
+
names = [row[0] for row in rows]
|
|
779
|
+
if BIGCAPITAL_SYSTEM_DB not in names or not all(SQL_NAME.fullmatch(name) for name in names):
|
|
780
|
+
raise Failed("the Bigcapital databases are missing or carry unexpected names")
|
|
781
|
+
return names
|
|
782
|
+
|
|
783
|
+
|
|
784
|
+
def fingerprint(container):
|
|
785
|
+
"""Exact row counts of every Bigcapital table and each tenant's ledger totals; every step compares them."""
|
|
786
|
+
schemas = bigcapital_schemas(container)
|
|
787
|
+
listed = ", ".join(f"'{name}'" for name in schemas)
|
|
788
|
+
tables = maria(container, "SELECT table_schema, table_name FROM information_schema.tables "
|
|
789
|
+
f"WHERE table_type = 'BASE TABLE' AND table_schema IN ({listed}) ORDER BY 1, 2")
|
|
790
|
+
if not tables or not all(len(t) == 2 and SQL_NAME.fullmatch(t[1]) for t in tables):
|
|
791
|
+
raise Failed("unexpected table names in the Bigcapital databases")
|
|
792
|
+
union = " UNION ALL ".join(f"SELECT '{s}.{t}', COUNT(*) FROM `{s}`.`{t}`" for s, t in tables)
|
|
793
|
+
rows = {name: int(count) for name, count in maria(container, union, 1800)}
|
|
794
|
+
ledgers = {}
|
|
795
|
+
for schema in schemas:
|
|
796
|
+
if [schema, "accounts_transactions"] in tables:
|
|
797
|
+
[[credit, debit]] = maria(container, "SELECT COALESCE(SUM(credit), 0), COALESCE(SUM(debit), 0) "
|
|
798
|
+
f"FROM `{schema}`.`accounts_transactions`")
|
|
799
|
+
ledgers[schema] = [credit, debit]
|
|
800
|
+
return {"schemas": schemas, "rows": rows, "ledgers": ledgers}
|
|
801
|
+
|
|
802
|
+
|
|
803
|
+
def summary(prints):
|
|
804
|
+
"""Counts only: the ledger totals stay in the root-only record."""
|
|
805
|
+
rows = prints["rows"]
|
|
806
|
+
tenant = [count for name, count in rows.items() if name.startswith(BIGCAPITAL_TENANT_PREFIX) and name.endswith(".knex_migrations")]
|
|
807
|
+
return {"databases": len(prints["schemas"]), "tables": len(rows), "rows": sum(rows.values()), "ledgers": len(prints["ledgers"]),
|
|
808
|
+
"migrations": {"system": rows.get(f"{BIGCAPITAL_SYSTEM_DB}.knex_migrations"), "tenant": sum(tenant)}}
|
|
809
|
+
|
|
810
|
+
|
|
811
|
+
def differences(before, after):
|
|
812
|
+
def changed(key):
|
|
813
|
+
old, new = before[key], after[key]
|
|
814
|
+
return sorted(name for name in set(old) | set(new) if old.get(name) != new.get(name))
|
|
815
|
+
return {"tables": changed("rows"), "ledgers": changed("ledgers")}
|
|
816
|
+
|
|
817
|
+
|
|
818
|
+
def write_json(path, value):
|
|
819
|
+
write_private(path, json.dumps(value, sort_keys=True, indent=1).encode())
|
|
820
|
+
|
|
821
|
+
|
|
822
|
+
def new_record():
|
|
823
|
+
root_dir(BIGCAPITAL_BACKUPS)
|
|
824
|
+
stamp = utc_stamp()
|
|
825
|
+
path = BIGCAPITAL_BACKUPS / stamp
|
|
826
|
+
if path.exists():
|
|
827
|
+
raise Failed(f"{path} already exists")
|
|
828
|
+
return stamp, root_dir(path)
|
|
829
|
+
|
|
830
|
+
|
|
831
|
+
def existing_record(stamp, usage):
|
|
832
|
+
path = BIGCAPITAL_BACKUPS / need(STAMP, stamp, usage)
|
|
833
|
+
if path.is_symlink() or not path.is_dir():
|
|
834
|
+
raise Failed(f"no Bigcapital record {stamp}")
|
|
835
|
+
return path
|
|
836
|
+
|
|
837
|
+
|
|
838
|
+
def bigcapital_images():
|
|
839
|
+
"""Every container of the Compose project with its configured image, image ID and registry digests."""
|
|
840
|
+
names = must([DOCKER, "ps", "-a", "--filter", f"label=com.docker.compose.project={BIGCAPITAL}", "--format", "{{.Names}}"],
|
|
841
|
+
30, "docker ps").split()
|
|
842
|
+
images = []
|
|
843
|
+
for name in sorted(names):
|
|
844
|
+
image, image_id = must([DOCKER, "inspect", "--format", "{{.Config.Image}} {{.Image}}", name], 30, "docker inspect").split()
|
|
845
|
+
digests = json.loads(must([DOCKER, "image", "inspect", "--format", "{{json .RepoDigests}}", image_id], 30, "docker image inspect"))
|
|
846
|
+
images.append({"container": name, "image": image, "image_id": image_id, "repo_digests": digests})
|
|
847
|
+
return images
|
|
848
|
+
|
|
849
|
+
|
|
850
|
+
def bigcapital_dump(path, schemas):
|
|
851
|
+
"""A consistent dump of the Bigcapital databases only: the system schema differs between server versions."""
|
|
852
|
+
target = path / BIGCAPITAL_DUMP
|
|
853
|
+
dump = ('exec "$(command -v mariadb-dump || command -v mysqldump)" -uroot -p"$MYSQL_ROOT_PASSWORD" '
|
|
854
|
+
'--single-transaction --routines --triggers --events --databases "$@"')
|
|
855
|
+
descriptor = os.open(target, os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_NOFOLLOW, 0o600)
|
|
856
|
+
with os.fdopen(descriptor, "wb") as raw, gzip.GzipFile(fileobj=raw, mode="wb") as out:
|
|
857
|
+
process = subprocess.Popen([DOCKER, "exec", BIGCAPITAL_DB, "sh", "-c", dump, "sh", *schemas],
|
|
858
|
+
stdout=subprocess.PIPE, stderr=subprocess.DEVNULL)
|
|
859
|
+
shutil.copyfileobj(process.stdout, out)
|
|
860
|
+
code = process.wait(timeout=3600)
|
|
861
|
+
if code != 0:
|
|
862
|
+
raise Failed(f"the dump failed (exit {code}); {target} is incomplete and has no manifest")
|
|
863
|
+
tail = b""
|
|
864
|
+
with gzip.open(target, "rb") as check:
|
|
865
|
+
for block in iter(lambda: check.read(1 << 20), b""):
|
|
866
|
+
tail = (tail + block)[-4096:]
|
|
867
|
+
if b"-- Dump completed" not in tail:
|
|
868
|
+
raise Failed(f"{target.name} lacks the dump's completion line; it has no manifest")
|
|
869
|
+
return target
|
|
870
|
+
|
|
871
|
+
|
|
872
|
+
def bigcapital_record(stamp, path, quiescent):
|
|
873
|
+
schemas = bigcapital_schemas(BIGCAPITAL_DB)
|
|
874
|
+
listed = ", ".join(f"'{name}'" for name in schemas)
|
|
875
|
+
[[size]] = maria(BIGCAPITAL_DB, f"SELECT COALESCE(SUM(data_length + index_length), 0) FROM information_schema.tables WHERE table_schema IN ({listed})")
|
|
876
|
+
if shutil.disk_usage(path).free < int(size):
|
|
877
|
+
raise Failed(f"less free space under {BIGCAPITAL_BACKUPS} than the databases' {size} bytes; nothing was dumped")
|
|
878
|
+
images = bigcapital_images()
|
|
879
|
+
write_json(path / "release-images.json", images)
|
|
880
|
+
prints = fingerprint(BIGCAPITAL_DB)
|
|
881
|
+
target = bigcapital_dump(path, schemas)
|
|
882
|
+
checksum = sha256_file(target)
|
|
883
|
+
write_private(path / "SHA256SUMS", f"{checksum} {target.name}\n".encode())
|
|
884
|
+
write_json(path / "fingerprint.json", prints)
|
|
885
|
+
info = {"stamp": stamp, "path": str(path), "file": target.name, "bytes": target.stat().st_size, "sha256": checksum,
|
|
886
|
+
"server_version": maria(BIGCAPITAL_DB, "SELECT VERSION()")[0][0], "quiescent": quiescent, "fingerprint": summary(prints),
|
|
887
|
+
"images": {i["container"]: i["image_id"] for i in images}}
|
|
888
|
+
write_json(path / "manifest.json", info)
|
|
889
|
+
return info
|
|
890
|
+
|
|
891
|
+
|
|
892
|
+
def op_bigcapital_backup(args):
|
|
893
|
+
"""Online and consistent (--single-transaction) while the app runs; the switch takes its own with the app stopped."""
|
|
894
|
+
arity(args, 0, 0, "bigcapital-backup")
|
|
895
|
+
if not container_running(BIGCAPITAL_DB):
|
|
896
|
+
raise Failed(f"{BIGCAPITAL_DB} is not running")
|
|
897
|
+
stamp, path = new_record()
|
|
898
|
+
emit(bigcapital_record(stamp, path, quiescent=False))
|
|
899
|
+
|
|
900
|
+
|
|
901
|
+
def pull_build(tag):
|
|
902
|
+
"""The image ID of each fork image at that tag; the rehearsal and the switch compare them."""
|
|
903
|
+
ids = {}
|
|
904
|
+
for service in BIGCAPITAL_SERVICES:
|
|
905
|
+
ref = BIGCAPITAL_IMAGE.format(service, tag)
|
|
906
|
+
must([DOCKER, "pull", "--quiet", ref], 900, f"docker pull {ref}")
|
|
907
|
+
ids[service] = must([DOCKER, "image", "inspect", "--format", "{{.Id}}", ref], 30, "docker image inspect").strip()
|
|
908
|
+
return ids
|
|
909
|
+
|
|
910
|
+
|
|
911
|
+
def restore_dump(container, source):
|
|
912
|
+
with gzip.open(source, "rb") as dump:
|
|
913
|
+
process = subprocess.Popen([DOCKER, "exec", "-i", container, "sh", "-c", SQL_CLIENT],
|
|
914
|
+
stdin=subprocess.PIPE, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
|
|
915
|
+
try:
|
|
916
|
+
shutil.copyfileobj(dump, process.stdin)
|
|
917
|
+
process.stdin.close()
|
|
918
|
+
except BrokenPipeError:
|
|
919
|
+
pass
|
|
920
|
+
code = process.wait(timeout=3600)
|
|
921
|
+
if code != 0:
|
|
922
|
+
raise Failed(f"the restore exited {code}; its output was withheld")
|
|
923
|
+
|
|
924
|
+
|
|
925
|
+
def wait_for_health(container, seconds):
|
|
926
|
+
deadline = time.monotonic() + seconds
|
|
927
|
+
health = "starting"
|
|
928
|
+
while time.monotonic() < deadline:
|
|
929
|
+
state, _, health = must([DOCKER, "inspect", "--format", "{{.State.Status}} {{if .State.Health}}{{.State.Health.Status}}{{end}}", container],
|
|
930
|
+
30, "docker inspect").strip().partition(" ")
|
|
931
|
+
if state != "running":
|
|
932
|
+
return state
|
|
933
|
+
if health in ("healthy", "unhealthy"):
|
|
934
|
+
return health
|
|
935
|
+
time.sleep(5)
|
|
936
|
+
return f"still {health or 'without a health check'} after {seconds} s"
|
|
937
|
+
|
|
938
|
+
|
|
939
|
+
# Error names and codes only, never message text: enough to tell a Node or driver failure apart.
|
|
940
|
+
DIAGNOSTIC = re.compile(r"\b(?:ER_[A-Z_]{2,60}|ERR_[A-Z0-9_]{2,60}|[A-Z][A-Za-z]{1,40}Error|NODE_MODULE_VERSION [0-9]{1,4}"
|
|
941
|
+
r"|E(?:CONNREFUSED|NOTFOUND|ACCES|NOENT|ADDRINUSE|TIMEDOUT)|Cannot find module '[@A-Za-z0-9_./-]{1,120}')")
|
|
942
|
+
|
|
943
|
+
|
|
944
|
+
def error_names(container):
|
|
945
|
+
result = run([DOCKER, "logs", "--tail", "400", container], 60)
|
|
946
|
+
text = (result.stdout + result.stderr).decode("utf-8", "replace")
|
|
947
|
+
return sorted(set(DIAGNOSTIC.findall(text)))[:20]
|
|
948
|
+
|
|
949
|
+
|
|
950
|
+
def op_bigcapital_rehearse(args):
|
|
951
|
+
"""Restores a backup into the current MariaDB on an internal network, upgrades it with the build's MariaDB,
|
|
952
|
+
runs the build's migrations, starts its server and Redis, and validates its proxy configuration; then
|
|
953
|
+
removes everything it created. A copy of the books never leaves the internal network."""
|
|
954
|
+
usage = "bigcapital-rehearse <stamp> <build>"
|
|
955
|
+
arity(args, 2, 2, usage)
|
|
956
|
+
build = need(BUILD, args[1], usage)
|
|
957
|
+
path = existing_record(args[0], usage)
|
|
958
|
+
source = path / BIGCAPITAL_DUMP
|
|
959
|
+
if source.is_symlink() or not source.is_file() or sha256_file(source) != (path / "SHA256SUMS").read_text(encoding="utf-8").split()[0]:
|
|
960
|
+
raise Failed("the backup's dump is missing or does not match its checksum")
|
|
961
|
+
record = path / f"rehearsal-{build}.json"
|
|
962
|
+
if record.exists():
|
|
963
|
+
raise Failed(f"{record.name} already exists; take a new backup to rehearse again")
|
|
964
|
+
current = must([DOCKER, "inspect", "--format", "{{.Image}}", BIGCAPITAL_DB], 30, "docker inspect").strip()
|
|
965
|
+
images = pull_build(build)
|
|
966
|
+
name = f"bigcapital-rehearsal-{build}"
|
|
967
|
+
db, migration, redis, server = (f"{name}-{part}" for part in ("db", "migration", "redis", "server"))
|
|
968
|
+
root_password, user_password, jwt_secret = (secrets.token_hex(24) for _ in range(3))
|
|
969
|
+
database = ["--network", name, "--network-alias", "mysql", "-v", f"{name}:/var/lib/mysql", "-e", f"MYSQL_ROOT_PASSWORD={root_password}"]
|
|
970
|
+
app_env = [part for key, value in {**BIGCAPITAL_SERVER_ENV, "DB_PASSWORD": user_password}.items() for part in ("-e", f"{key}={value}")]
|
|
971
|
+
checks = []
|
|
972
|
+
outcome = {"stamp": args[0], "build": build, "images": images, "current_mysql_image": current, "checks": checks}
|
|
973
|
+
try:
|
|
974
|
+
must([DOCKER, "network", "create", "--internal", name], 60, "docker network create")
|
|
975
|
+
must([DOCKER, "volume", "create", name], 60, "docker volume create")
|
|
976
|
+
must([DOCKER, "run", "-d", "--name", db, *database, "-e", f"MYSQL_DATABASE={BIGCAPITAL_SYSTEM_DB}", "-e", "MYSQL_USER=bigcapital",
|
|
977
|
+
"-e", f"MYSQL_PASSWORD={user_password}", current], 120, "docker run of the current MariaDB")
|
|
978
|
+
wait_for_server(db, 300)
|
|
979
|
+
restore_dump(db, source)
|
|
980
|
+
restored = fingerprint(db)
|
|
981
|
+
must([DOCKER, "stop", "-t", "300", db], 330, "docker stop of the current MariaDB")
|
|
982
|
+
must([DOCKER, "rm", db], 60, "docker rm")
|
|
983
|
+
must([DOCKER, "run", "-d", "--name", db, *database, "-e", "MARIADB_AUTO_UPGRADE=1", images["mysql"]], 120, "docker run of the new MariaDB")
|
|
984
|
+
wait_for_server(db, 900)
|
|
985
|
+
upgraded = fingerprint(db)
|
|
986
|
+
version = maria(db, "SELECT VERSION()")[0][0]
|
|
987
|
+
checks.append({"check": "upgrade kept every row and ledger", "ok": upgraded == restored,
|
|
988
|
+
"observed": {"server_version": version, **differences(restored, upgraded)}})
|
|
989
|
+
code = run([DOCKER, "run", "--name", migration, "--network", name, *app_env, images["migration"]], 1800).returncode
|
|
990
|
+
checks.append({"check": "migrations exit 0", "ok": code == 0, "observed": code if code == 0 else {"exit": code, "errors": error_names(migration)}})
|
|
991
|
+
migrated = fingerprint(db)
|
|
992
|
+
changes = differences(upgraded, migrated)
|
|
993
|
+
checks.append({"check": "migrations kept every ledger", "ok": not changes["ledgers"], "observed": changes})
|
|
994
|
+
must([DOCKER, "run", "-d", "--name", redis, "--network", name, "--network-alias", "redis", images["redis"]], 120, "docker run of the new Redis")
|
|
995
|
+
must([DOCKER, "run", "-d", "--name", server, "--network", name, *app_env, "-e", f"APP_JWT_SECRET={jwt_secret}", images["server"]],
|
|
996
|
+
120, "docker run of the new server")
|
|
997
|
+
health = wait_for_health(server, 300)
|
|
998
|
+
checks.append({"check": "server healthy", "ok": health == "healthy",
|
|
999
|
+
"observed": health if health == "healthy" else {"health": health, "errors": error_names(server)}})
|
|
1000
|
+
pong = run([DOCKER, "exec", redis, "redis-cli", "ping"], 30).stdout.strip().decode("utf-8", "replace")
|
|
1001
|
+
checks.append({"check": "redis answers", "ok": pong == "PONG", "observed": pong[:40]})
|
|
1002
|
+
code = run([DOCKER, "run", "--rm", "--network", "none", images["proxy"], "envoy", "--mode", "validate", "-c", "/etc/envoy/envoy.yaml"],
|
|
1003
|
+
120).returncode
|
|
1004
|
+
checks.append({"check": "proxy configuration valid", "ok": code == 0, "observed": code})
|
|
1005
|
+
outcome["fingerprint"] = summary(migrated)
|
|
1006
|
+
except Failed as error:
|
|
1007
|
+
checks.append({"check": "completed", "ok": False, "observed": str(error)})
|
|
1008
|
+
finally:
|
|
1009
|
+
for container in (server, redis, migration, db):
|
|
1010
|
+
run([DOCKER, "rm", "-f", "-v", container], 120)
|
|
1011
|
+
run([DOCKER, "volume", "rm", name], 120)
|
|
1012
|
+
run([DOCKER, "network", "rm", name], 60)
|
|
1013
|
+
outcome["ok"] = all(c["ok"] for c in checks)
|
|
1014
|
+
write_json(record, outcome)
|
|
1015
|
+
emit(outcome)
|
|
1016
|
+
if not outcome["ok"]:
|
|
1017
|
+
raise Failed("the rehearsal failed; see the JSON above")
|
|
1018
|
+
|
|
1019
|
+
|
|
1020
|
+
def volume_of(container, destination):
|
|
1021
|
+
mounts = json.loads(must([DOCKER, "inspect", "--format", "{{json .Mounts}}", container], 30, "docker inspect"))
|
|
1022
|
+
for mount in mounts:
|
|
1023
|
+
if mount.get("Type") == "volume" and mount.get("Destination") == destination:
|
|
1024
|
+
return mount["Name"]
|
|
1025
|
+
raise Failed(f"{container} mounts no volume at {destination}")
|
|
1026
|
+
|
|
1027
|
+
|
|
1028
|
+
def mountpoint(volume):
|
|
1029
|
+
return Path(must([DOCKER, "volume", "inspect", "--format", "{{.Mountpoint}}", volume], 30, "docker volume inspect").strip())
|
|
1030
|
+
|
|
1031
|
+
|
|
1032
|
+
def tree(root):
|
|
1033
|
+
"""Type, mode, owner, and a file's size and content for every entry: the copy must match the source exactly.
|
|
1034
|
+
|
|
1035
|
+
A directory's own size depends on its history, so it is left out.
|
|
1036
|
+
"""
|
|
1037
|
+
top = root.lstat()
|
|
1038
|
+
entries = {".": (top.st_mode, top.st_uid, top.st_gid, None, None)}
|
|
1039
|
+
for current, dirs, files in os.walk(root):
|
|
1040
|
+
dirs.sort()
|
|
1041
|
+
for name in dirs + sorted(files):
|
|
1042
|
+
full = Path(current) / name
|
|
1043
|
+
info = full.lstat()
|
|
1044
|
+
if stat.S_ISREG(info.st_mode):
|
|
1045
|
+
size, content = info.st_size, sha256_file(full)
|
|
1046
|
+
else:
|
|
1047
|
+
size, content = None, os.readlink(full) if stat.S_ISLNK(info.st_mode) else None
|
|
1048
|
+
entries[str(full.relative_to(root))] = (info.st_mode, info.st_uid, info.st_gid, size, content)
|
|
1049
|
+
return entries
|
|
1050
|
+
|
|
1051
|
+
|
|
1052
|
+
def tree_bytes(entries):
|
|
1053
|
+
return sum(size for _, _, _, size, _ in entries.values() if size is not None)
|
|
1054
|
+
|
|
1055
|
+
|
|
1056
|
+
def copy_volume(source, target):
|
|
1057
|
+
"""cp -a into the new volume, its top directory's owner and mode, then a full comparison."""
|
|
1058
|
+
must(["/bin/cp", "-a", "--", f"{source}/.", f"{target}/"], 1800, "cp -a")
|
|
1059
|
+
top = source.lstat()
|
|
1060
|
+
os.chown(target, top.st_uid, top.st_gid)
|
|
1061
|
+
os.chmod(target, stat.S_IMODE(top.st_mode))
|
|
1062
|
+
expected = tree(source)
|
|
1063
|
+
if tree(target) != expected:
|
|
1064
|
+
raise Failed(f"the copy of {source} differs from its source")
|
|
1065
|
+
return {"entries": len(expected), "bytes": tree_bytes(expected)}
|
|
1066
|
+
|
|
1067
|
+
|
|
1068
|
+
def op_bigcapital_switch(args):
|
|
1069
|
+
"""Owner-approved. With a passed rehearsal of the build the astrofoundry tags now name, stops the app,
|
|
1070
|
+
takes the quiescent dump, saves Redis, stops both, and copies their volumes to new ones. The stack stays
|
|
1071
|
+
stopped for the GitOps change that points it at the copies; the old volumes stay as they were for a
|
|
1072
|
+
rollback. A failure before the end starts the stopped containers again."""
|
|
1073
|
+
usage = "bigcapital-switch <stamp> <build>"
|
|
1074
|
+
arity(args, 2, 2, usage)
|
|
1075
|
+
build = need(BUILD, args[1], usage)
|
|
1076
|
+
path = existing_record(args[0], usage)
|
|
1077
|
+
record = path / f"rehearsal-{build}.json"
|
|
1078
|
+
if record.is_symlink() or not record.is_file() or json.loads(record.read_text(encoding="utf-8")).get("ok") is not True:
|
|
1079
|
+
raise Failed(f"no passed rehearsal of {build} on backup {args[0]}; nothing was stopped")
|
|
1080
|
+
rehearsed = json.loads(record.read_text(encoding="utf-8"))["images"]
|
|
1081
|
+
built = pull_build(build)
|
|
1082
|
+
if built != rehearsed:
|
|
1083
|
+
raise Failed(f"the images of {build} changed since the rehearsal; nothing was stopped")
|
|
1084
|
+
if pull_build("astrofoundry") != built:
|
|
1085
|
+
raise Failed(f"the astrofoundry tags do not name build {build}; nothing was stopped")
|
|
1086
|
+
for container in (BIGCAPITAL_DB, BIGCAPITAL_REDIS, *BIGCAPITAL_APP):
|
|
1087
|
+
if not container_running(container):
|
|
1088
|
+
raise Failed(f"{container} is not running; nothing was stopped")
|
|
1089
|
+
sources = {"mysql": volume_of(BIGCAPITAL_DB, "/var/lib/mysql"), "redis": volume_of(BIGCAPITAL_REDIS, "/data")}
|
|
1090
|
+
needed = sum(tree_bytes(tree(mountpoint(volume))) for volume in sources.values())
|
|
1091
|
+
if shutil.disk_usage(mountpoint(sources["mysql"])).free < 2 * needed:
|
|
1092
|
+
raise Failed(f"less than twice the volumes' {needed} bytes free; nothing was stopped")
|
|
1093
|
+
stamp, switch = new_record()
|
|
1094
|
+
targets = {kind: f"bigcapital_{kind}_{stamp}" for kind in sources}
|
|
1095
|
+
stopped = []
|
|
1096
|
+
try:
|
|
1097
|
+
for container in BIGCAPITAL_APP:
|
|
1098
|
+
must([DOCKER, "stop", "-t", "60", container], 90, f"docker stop {container}")
|
|
1099
|
+
stopped.append(container)
|
|
1100
|
+
backup = bigcapital_record(stamp, switch, quiescent=True)
|
|
1101
|
+
if run([DOCKER, "exec", BIGCAPITAL_REDIS, "redis-cli", "SAVE"], 300).stdout.strip() != b"OK":
|
|
1102
|
+
raise Failed("redis-cli SAVE did not answer OK")
|
|
1103
|
+
for container in (BIGCAPITAL_DB, BIGCAPITAL_REDIS):
|
|
1104
|
+
must([DOCKER, "stop", "-t", "300", container], 330, f"docker stop {container}")
|
|
1105
|
+
stopped.append(container)
|
|
1106
|
+
copies = {}
|
|
1107
|
+
for kind, volume in sources.items():
|
|
1108
|
+
must([DOCKER, "volume", "create", targets[kind]], 60, "docker volume create")
|
|
1109
|
+
copies[kind] = {"from": volume, "to": targets[kind], **copy_volume(mountpoint(volume), mountpoint(targets[kind]))}
|
|
1110
|
+
except Exception as error:
|
|
1111
|
+
for container in reversed(stopped):
|
|
1112
|
+
run([DOCKER, "start", container], 120)
|
|
1113
|
+
detail = str(error) if isinstance(error, Failed) else type(error).__name__
|
|
1114
|
+
raise Failed(f"{detail}; the stopped containers were started again and the old volumes are unchanged") from None
|
|
1115
|
+
outcome = {"stamp": stamp, "path": str(switch), "build": build, "rehearsal": args[0], "images": built, "volumes": copies,
|
|
1116
|
+
"backup": {k: backup[k] for k in ("file", "sha256", "server_version", "fingerprint")}, "stopped": stopped}
|
|
1117
|
+
write_json(switch / "switch.json", outcome)
|
|
1118
|
+
emit(outcome)
|
|
1119
|
+
|
|
1120
|
+
|
|
1121
|
+
def op_bigcapital_verify(args):
|
|
1122
|
+
"""After the GitOps change: the copies are mounted, the build runs, and the books match the switch's dump."""
|
|
1123
|
+
usage = "bigcapital-verify <stamp>"
|
|
1124
|
+
arity(args, 1, 1, usage)
|
|
1125
|
+
path = existing_record(args[0], usage)
|
|
1126
|
+
if not (path / "switch.json").is_file():
|
|
1127
|
+
raise Failed(f"{args[0]} is not a completed switch")
|
|
1128
|
+
switch = json.loads((path / "switch.json").read_text(encoding="utf-8"))
|
|
1129
|
+
before = json.loads((path / "fingerprint.json").read_text(encoding="utf-8"))
|
|
1130
|
+
checks = []
|
|
1131
|
+
for kind, container, destination in (("mysql", BIGCAPITAL_DB, "/var/lib/mysql"), ("redis", BIGCAPITAL_REDIS, "/data")):
|
|
1132
|
+
observed = volume_of(container, destination)
|
|
1133
|
+
checks.append({"check": f"{kind} mounts its copy", "ok": observed == switch["volumes"][kind]["to"], "observed": observed})
|
|
1134
|
+
for service in ("mysql", "redis", "server", "webapp", "proxy", "migration"):
|
|
1135
|
+
container = f"{BIGCAPITAL}-{'database_migration' if service == 'migration' else service}-1"
|
|
1136
|
+
observed = run([DOCKER, "inspect", "--format", "{{.Image}}", container], 30).stdout.strip().decode()
|
|
1137
|
+
checks.append({"check": f"{service} runs the build", "ok": observed == switch["images"][service], "observed": observed[:19]})
|
|
1138
|
+
exit_code = run([DOCKER, "inspect", "--format", "{{.State.ExitCode}}", f"{BIGCAPITAL}-database_migration-1"], 30).stdout.strip().decode()
|
|
1139
|
+
checks.append({"check": "migrations exit 0", "ok": exit_code == "0", "observed": exit_code})
|
|
1140
|
+
health = wait_for_health("bigcapital-server-1", 300)
|
|
1141
|
+
checks.append({"check": "server healthy", "ok": health == "healthy", "observed": health})
|
|
1142
|
+
after = fingerprint(BIGCAPITAL_DB)
|
|
1143
|
+
changes = differences(before, after)
|
|
1144
|
+
checks.append({"check": "ledgers match the switch's dump", "ok": not changes["ledgers"], "observed": changes["ledgers"]})
|
|
1145
|
+
checks.append({"check": "row counts match the switch's dump", "ok": not changes["tables"], "observed": changes["tables"][:20]})
|
|
1146
|
+
emit({"ok": all(c["ok"] for c in checks), "server_version": maria(BIGCAPITAL_DB, "SELECT VERSION()")[0][0],
|
|
1147
|
+
"fingerprint": summary(after), "checks": checks})
|
|
1148
|
+
|
|
1149
|
+
|
|
719
1150
|
COMMANDS = {
|
|
720
1151
|
"container-memory": op_container_memory,
|
|
721
1152
|
"container-stats": op_container_stats,
|
|
@@ -748,6 +1179,10 @@ COMMANDS = {
|
|
|
748
1179
|
"db-dump": op_db_dump,
|
|
749
1180
|
"db-restore": op_db_restore,
|
|
750
1181
|
"supabase-backup": op_supabase_backup,
|
|
1182
|
+
"bigcapital-backup": op_bigcapital_backup,
|
|
1183
|
+
"bigcapital-rehearse": op_bigcapital_rehearse,
|
|
1184
|
+
"bigcapital-switch": op_bigcapital_switch,
|
|
1185
|
+
"bigcapital-verify": op_bigcapital_verify,
|
|
751
1186
|
}
|
|
752
1187
|
|
|
753
1188
|
|
|
@@ -28,6 +28,8 @@ describe("Arcane host commands", () => {
|
|
|
28
28
|
expect(buildHostRemote(["db-restore", "mealie", "20261019T120000Z"]).name).toBe("db-restore");
|
|
29
29
|
expect(buildHostRemote(["mealie-api"]).spec.stdinSecret).toBe("MEALIE_API_KEY");
|
|
30
30
|
expect(buildHostRemote(["supabase-backup"]).remote).toBe("supabase-backup");
|
|
31
|
+
expect(buildHostRemote(["bigcapital-rehearse", "20261104T080000Z", "a87df4f"]).remote).toBe("bigcapital-rehearse 20261104T080000Z a87df4f");
|
|
32
|
+
expect(buildHostRemote(["bigcapital-verify", "20261104T090000Z"]).remote).toBe("bigcapital-verify 20261104T090000Z");
|
|
31
33
|
const bad = [
|
|
32
34
|
[],
|
|
33
35
|
["rm"],
|
|
@@ -43,6 +45,10 @@ describe("Arcane host commands", () => {
|
|
|
43
45
|
["db-dump", "supabase"],
|
|
44
46
|
["db-restore", "mealie", "latest"],
|
|
45
47
|
["supabase-backup", "_supabase"],
|
|
48
|
+
["bigcapital-backup", "now"],
|
|
49
|
+
["bigcapital-rehearse", "20261104T080000Z", "astrofoundry"],
|
|
50
|
+
["bigcapital-switch", "latest", "a87df4f"],
|
|
51
|
+
["bigcapital-verify", "20261104T090000Z", "a87df4f"],
|
|
46
52
|
["container-stats", ...Array.from({ length: 9 }, (_, i) => `c${i}`)],
|
|
47
53
|
["manager-start"],
|
|
48
54
|
["manager-rollback", "20261019T120000Z"],
|
|
@@ -47,6 +47,10 @@ export const HOST_COMMANDS: Readonly<Record<string, HostCommand>> = {
|
|
|
47
47
|
"db-dump": { args: [1, 1], help: "db-dump mealie|paperless pg_dump into /var/backups/spc-arcane/<stack>/<UTC> (gzip, checked, sha256)", timeoutMs: 60 * MINUTE },
|
|
48
48
|
"db-restore": { args: [2, 2], help: "db-restore <stack> <stamp> fill an EMPTY database from one of these dumps, the app stopped meanwhile (owner approval)", timeoutMs: 60 * MINUTE },
|
|
49
49
|
"supabase-backup": { args: [0, 0], help: "supabase-backup both Supabase databases, roles, and db-config (pgsodium key) into /var/backups/supabase-upgrades/<UTC>, checked", timeoutMs: 90 * MINUTE },
|
|
50
|
+
"bigcapital-backup": { args: [0, 0], help: "bigcapital-backup online dump of the Bigcapital databases, row counts, ledger totals, image digests into /var/backups/bigcapital-upgrades/<UTC>", timeoutMs: 60 * MINUTE },
|
|
51
|
+
"bigcapital-rehearse": { args: [2, 2], help: "bigcapital-rehearse <stamp> <build> restore that backup on an internal network, upgrade it with the build's MariaDB, run its migrations, start its server", timeoutMs: 120 * MINUTE },
|
|
52
|
+
"bigcapital-switch": { args: [2, 2], help: "bigcapital-switch <stamp> <build> stop Bigcapital, quiescent dump, copy its volumes to new ones for the GitOps switch (owner approval)", timeoutMs: 90 * MINUTE },
|
|
53
|
+
"bigcapital-verify": { args: [1, 1], help: "bigcapital-verify <stamp> after the switch: copied volumes mounted, the build running, books matching the switch's dump", timeoutMs: 15 * MINUTE },
|
|
50
54
|
"manager-state": { args: [0, 0], help: "manager-state", timeoutMs: MINUTE, internal: true },
|
|
51
55
|
"manager-prepare": { args: [1, 1], help: "manager-prepare <stamp>", timeoutMs: 2 * MINUTE, internal: true },
|
|
52
56
|
"manager-pull": { args: [1, 1], help: "manager-pull <digest>", timeoutMs: 16 * MINUTE, internal: true },
|
|
@@ -59,6 +63,8 @@ export const HOST_COMMANDS: Readonly<Record<string, HostCommand>> = {
|
|
|
59
63
|
const NAME = /^[A-Za-z0-9][A-Za-z0-9_.-]{0,127}$/;
|
|
60
64
|
const UTC = /^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}Z$/;
|
|
61
65
|
export const STAMP = /^\d{8}T\d{6}Z$/;
|
|
66
|
+
/** A fork build: the seven-character commit tag the bigcapital CI publishes. */
|
|
67
|
+
const BUILD = /^[0-9a-f]{7}$/;
|
|
62
68
|
const IMDB = /^tt\d{1,12}$/;
|
|
63
69
|
const UUID = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/;
|
|
64
70
|
const OBJECT_PATH = /^[A-Za-z0-9_][A-Za-z0-9_./-]{0,255}$/;
|
|
@@ -120,6 +126,14 @@ export function buildHostRemote(args: string[], allowInternal = false): { remote
|
|
|
120
126
|
check((DB_STACKS as readonly string[]).includes(rest[0]), `stack: ${DB_STACKS.join(", ")}`);
|
|
121
127
|
check(STAMP.test(rest[1]), "stamp: YYYYMMDDTHHMMSSZ from db-list");
|
|
122
128
|
break;
|
|
129
|
+
case "bigcapital-rehearse":
|
|
130
|
+
case "bigcapital-switch":
|
|
131
|
+
check(STAMP.test(rest[0]), "stamp: YYYYMMDDTHHMMSSZ from bigcapital-backup");
|
|
132
|
+
check(BUILD.test(rest[1]), "build: the seven-character commit tag of the fork's images");
|
|
133
|
+
break;
|
|
134
|
+
case "bigcapital-verify":
|
|
135
|
+
check(STAMP.test(rest[0]), "stamp: YYYYMMDDTHHMMSSZ from bigcapital-switch");
|
|
136
|
+
break;
|
|
123
137
|
default:
|
|
124
138
|
break;
|
|
125
139
|
}
|