@astrofoundry/pi-astro 0.40.49 → 0.40.50
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
|
@@ -57,7 +57,7 @@ export function edgeStatus(status: string | null, updates: string | null): AreaS
|
|
|
57
57
|
const parts = [
|
|
58
58
|
down.length === 0 ? `${units.length} services active` : `down: ${down.join(", ")}`,
|
|
59
59
|
`WireGuard ${handshake || "unknown"}`,
|
|
60
|
-
rows.get("reboot required") === "yes" ? "reboot pending (it reboots itself at
|
|
60
|
+
rows.get("reboot required") === "yes" ? "reboot pending (it reboots itself at 01:20 UTC)" : "no reboot pending",
|
|
61
61
|
upgradable === null ? "updates unknown" : `${upgradable} upgradable package${upgradable === 1 ? "" : "s"}`,
|
|
62
62
|
];
|
|
63
63
|
const tunnel = age === null || age > STALE_HANDSHAKE_S;
|
|
@@ -171,7 +171,7 @@ export function guestOwner(name: string): string {
|
|
|
171
171
|
return "Proxmox";
|
|
172
172
|
}
|
|
173
173
|
|
|
174
|
-
/** Each guest's and Synapse's `updates` after the
|
|
174
|
+
/** Each guest's and Synapse's `updates` after the 02:25 run: anything still waiting did not install and needs a look. */
|
|
175
175
|
export function updatesStatus(hosts: Readonly<Record<string, string | null>>): AreaStatus {
|
|
176
176
|
const names = Object.keys(hosts);
|
|
177
177
|
const wazuh = wazuhRelease(hosts);
|
|
@@ -319,7 +319,7 @@ export function mediaStatus(sab: unknown): AreaStatus {
|
|
|
319
319
|
return { area: "Media", line, ...(warnings > 0 ? { problem: `Media: the downloader shows ${warnings} warning${warnings === 1 ? "" : "s"}`, route: "media" as const } : {}) };
|
|
320
320
|
}
|
|
321
321
|
|
|
322
|
-
/** Every guest is backed up nightly
|
|
322
|
+
/** Every guest is backed up nightly from 02:00; an archive older than this means last night's run missed it. */
|
|
323
323
|
const MISSED_BACKUP_S = 30 * HOUR;
|
|
324
324
|
|
|
325
325
|
/** `backup vzdump-archives` and `restic-check-status`: last night's Proxmox archives and the weekly restic check. */
|
package/package.json
CHANGED
package/skills/backup/SKILL.md
CHANGED
|
@@ -7,12 +7,12 @@ description: How the backup specialist checks and runs the homelab backup layers
|
|
|
7
7
|
|
|
8
8
|
Four layers protect the homelab. Every command is a fixed operation on Pulsar (root) or on the Arcane VM (user `arcane`); `["--help"]` lists them.
|
|
9
9
|
|
|
10
|
-
| Layer | What | Schedule | Commands |
|
|
10
|
+
| Layer | What | Schedule (Europe/Madrid) | Commands |
|
|
11
11
|
|---|---|---|---|
|
|
12
|
-
| 1 | vzdump of every guest to PVE storage `unas-backup` (NAS `//10.0.50.50/Shared_Drive/BACKUP/pulsar/dump`), snapshot mode, zstd, 7 daily + 4 weekly | 02:
|
|
13
|
-
| 2 | App-consistent service backups from the Arcane VM into the shared Restic repository (`/mnt/backup/restic` there), one tag per service, 30 daily + 12 monthly |
|
|
14
|
-
| 3 | Pulsar host config, Technitium ZIP exports, Identity `pg_dump` into Restic, tag `pulsar-host` |
|
|
15
|
-
| 4 | Offsite copy of the Restic repository to `gs://ardilla-backup/pulsar/restic` from LXC 102 (append-only service account) |
|
|
12
|
+
| 1 | vzdump of every guest to PVE storage `unas-backup` (NAS `//10.0.50.50/Shared_Drive/BACKUP/pulsar/dump`), snapshot mode, zstd, 7 daily + 4 weekly | 02:00 | `vzdump-jobs`, `vzdump-archives`, `vzdump-tasks`, `vzdump-run <vmid>` |
|
|
13
|
+
| 2 | App-consistent service backups from the Arcane VM into the shared Restic repository (`/mnt/backup/restic` there), one tag per service, 30 daily + 12 monthly | 02:15, check Sunday 02:25 | `arcane-backup-status|run|journal`, `restic-check-status|run|journal` |
|
|
14
|
+
| 3 | Pulsar host config, Technitium ZIP exports, Identity `pg_dump` into Restic, tag `pulsar-host` | 02:20 | `host-backup-status|run|journal` |
|
|
15
|
+
| 4 | Offsite copy of the Restic repository to `gs://ardilla-backup/pulsar/restic` from LXC 102 (append-only service account) | 03:10 | `gcs-status|run|journal` |
|
|
16
16
|
|
|
17
17
|
Restic commands run on Pulsar against `/mnt/pve/unas-backup/restic` with the host's password file: `restic-snapshots [tag]`, `restic-stats [tag]`, `restic-ls <snapshot> [path]`, `restic-restore <snapshot> <path>`, `restore-ls`, `restore-clean`. `restic-restore` writes under `/var/tmp/spc-restore/<snapshot>/` on Pulsar and nowhere else.
|
|
18
18
|
|
|
@@ -45,7 +45,7 @@ A service's backup is confirmed when `arcane-backup-status` shows `Result=succes
|
|
|
45
45
|
|
|
46
46
|
## Mira guests
|
|
47
47
|
|
|
48
|
-
SPACE-X (Mira) writes its own vzdump archives to TESLA (`tesla-backup`, `BACKUP/space-x/dump`), and Pulsar's `mira-backup-pull.timer` copies them to the UNAS (`BACKUP/space-x/dump`) at 05
|
|
48
|
+
SPACE-X (Mira) writes its own vzdump archives to TESLA (`tesla-backup`, `BACKUP/space-x/dump`), and Pulsar's `mira-backup-pull.timer` copies them to the UNAS (`BACKUP/space-x/dump`) at 03:05 without mirroring Mira's deletions. This tool does not read SPACE-X; the proxmox specialist's `space-x tasks` shows the runs, and Kuma monitors `space-x-backup` and `mira-backup-pull` carry the heartbeats.
|
|
49
49
|
|
|
50
50
|
## Reading the output
|
|
51
51
|
|
package/skills/edge/SKILL.md
CHANGED
|
@@ -48,7 +48,7 @@ Removing a hostname is the same with a removed line; check first that no public
|
|
|
48
48
|
|
|
49
49
|
## Vulnerability on the VPS
|
|
50
50
|
|
|
51
|
-
Wazuh reports a CVE against a package (the security specialist passes `package`, installed `version`, and `condition`, the fixed version). The VPS patches itself: unattended-upgrades installs Debian, Debian-Security,
|
|
51
|
+
Wazuh reports a CVE against a package (the security specialist passes `package`, installed `version`, and `condition`, the fixed version). The VPS patches itself: unattended-upgrades installs Debian, Debian-Security, `trixie-updates`, Google guest-agent, Google Cloud CLI, and CrowdSec bouncer updates at 02:05 Europe/Madrid, after a package-list refresh at 02:00, and reboots at 01:20 UTC when a package needs it (`00-frontdoor-vps/system/52unattended-upgrades-frontdoor`). Answer which of three states applies:
|
|
52
52
|
|
|
53
53
|
1. `package <name>`, then `changelog <name>`: if the installed version comes from the security origin and its changelog entry names the CVE, it is patched; say by which upload and, from `auto-upgrades` or the dpkg log, when. Wazuh may keep flagging the new version until its feed learns the fixed version; that is feed lag, not an open hole.
|
|
54
54
|
2. Otherwise, if the candidate's changelog names the CVE, it is scheduled: `auto-upgrades` shows when `apt-daily-upgrade.timer` runs next. Say so with the time.
|
package/skills/security/SKILL.md
CHANGED
|
@@ -64,7 +64,7 @@ The `security` tool is read-only. Every call is a fixed command on the guest; `[
|
|
|
64
64
|
|
|
65
65
|
You observe; the owning specialist acts. When a finding concerns a host in the table, call that specialist through `subagent` with the facts (`agent`, `package`, installed `version`, `condition`, `cve`) and a precise question, then report its answer with your own. Do not tell the caller to run commands on a host that has an owner.
|
|
66
66
|
|
|
67
|
-
Debian guests and Synapse install every stable update each night (
|
|
67
|
+
Debian guests and Synapse install every stable update each night (02:25 Europe/Madrid, technitium2 02:30, the Mira guests 02:35, each five minutes after a package-list refresh) and reboot right after when a package needs it; `wazuh-agent` is held. The VPS installs Debian, Debian-Security, `trixie-updates`, Google guest-agent, Google Cloud CLI, and CrowdSec bouncer updates at 02:05 Europe/Madrid and reboots at 01:20 UTC when needed. The morning run upgrades the Pulsar and SPACE-X hosts and offers the owner a Reboot button. For a vulnerability alert, ask the owner whether the package is already upgraded, will be at the next run (the fix is in the index), or is stuck (the fix is not in the index or the run failed), and to confirm from the package changelog that the installed version names the CVE. Only the stuck case needs the operator; say so plainly, with the versions. Answer with one of: patched (version, when, by what), scheduled (next run time), stuck (why), or feed lag (patched, Wazuh not updated yet).
|
|
68
68
|
|
|
69
69
|
## Operator actions (describe, do not perform)
|
|
70
70
|
|
|
@@ -792,10 +792,10 @@ def fingerprint(container):
|
|
|
792
792
|
union = " UNION ALL ".join(f"SELECT '{s}.{t}', COUNT(*) FROM `{s}`.`{t}`" for s, t in tables)
|
|
793
793
|
rows = {name: int(count) for name, count in maria(container, union, 1800)}
|
|
794
794
|
ledgers = {}
|
|
795
|
-
|
|
796
|
-
|
|
797
|
-
|
|
798
|
-
|
|
795
|
+
# Bigcapital's knex mappers upper-case its table names (ACCOUNTS_TRANSACTIONS), but not knex_migrations.
|
|
796
|
+
for schema, table in tables:
|
|
797
|
+
if table.lower() == "accounts_transactions":
|
|
798
|
+
[[credit, debit]] = maria(container, f"SELECT COALESCE(SUM(credit), 0), COALESCE(SUM(debit), 0) FROM `{schema}`.`{table}`")
|
|
799
799
|
ledgers[schema] = [credit, debit]
|
|
800
800
|
return {"schemas": schemas, "rows": rows, "ledgers": ledgers}
|
|
801
801
|
|
|
@@ -947,6 +947,18 @@ def error_names(container):
|
|
|
947
947
|
return sorted(set(DIAGNOSTIC.findall(text)))[:20]
|
|
948
948
|
|
|
949
949
|
|
|
950
|
+
def server_errors(container):
|
|
951
|
+
"""MariaDB's and its entrypoint's own error lines, without repeats; they name a failed step, not the books."""
|
|
952
|
+
result = run([DOCKER, "logs", "--tail", "400", container], 60)
|
|
953
|
+
lines = []
|
|
954
|
+
for line in (result.stdout + result.stderr).decode("utf-8", "replace").splitlines():
|
|
955
|
+
found = re.search(r"\[ERROR\].*|ERROR [0-9]{4} \([0-9A-Z]{5}\).*", line)
|
|
956
|
+
text = found.group(0)[:200] if found else None
|
|
957
|
+
if text and text not in lines:
|
|
958
|
+
lines.append(text)
|
|
959
|
+
return lines[-6:]
|
|
960
|
+
|
|
961
|
+
|
|
950
962
|
def op_bigcapital_rehearse(args):
|
|
951
963
|
"""Restores a backup into the current MariaDB on an internal network, upgrades it with the build's MariaDB,
|
|
952
964
|
runs the build's migrations, starts its server and Redis, and validates its proxy configuration; then
|
|
@@ -966,26 +978,32 @@ def op_bigcapital_rehearse(args):
|
|
|
966
978
|
name = f"bigcapital-rehearsal-{build}"
|
|
967
979
|
db, migration, redis, server = (f"{name}-{part}" for part in ("db", "migration", "redis", "server"))
|
|
968
980
|
root_password, user_password, jwt_secret = (secrets.token_hex(24) for _ in range(3))
|
|
969
|
-
database
|
|
981
|
+
# The stack's database settings: the image's own MYSQL_DATABASE default names no database, and the
|
|
982
|
+
# upgrade's healthcheck-user step fails on it.
|
|
983
|
+
database = ["--network", name, "--network-alias", "mysql", "-v", f"{name}:/var/lib/mysql", "-e", f"MYSQL_ROOT_PASSWORD={root_password}",
|
|
984
|
+
"-e", f"MYSQL_DATABASE={BIGCAPITAL_SYSTEM_DB}", "-e", "MYSQL_USER=bigcapital", "-e", f"MYSQL_PASSWORD={user_password}"]
|
|
970
985
|
app_env = [part for key, value in {**BIGCAPITAL_SERVER_ENV, "DB_PASSWORD": user_password}.items() for part in ("-e", f"{key}={value}")]
|
|
971
986
|
checks = []
|
|
972
987
|
outcome = {"stamp": args[0], "build": build, "images": images, "current_mysql_image": current, "checks": checks}
|
|
988
|
+
step = "network and volume"
|
|
973
989
|
try:
|
|
974
990
|
must([DOCKER, "network", "create", "--internal", name], 60, "docker network create")
|
|
975
991
|
must([DOCKER, "volume", "create", name], 60, "docker volume create")
|
|
976
|
-
|
|
977
|
-
|
|
992
|
+
step = "restore into the current MariaDB"
|
|
993
|
+
must([DOCKER, "run", "-d", "--name", db, *database, current], 120, "docker run of the current MariaDB")
|
|
978
994
|
wait_for_server(db, 300)
|
|
979
995
|
restore_dump(db, source)
|
|
980
996
|
restored = fingerprint(db)
|
|
981
997
|
must([DOCKER, "stop", "-t", "300", db], 330, "docker stop of the current MariaDB")
|
|
982
998
|
must([DOCKER, "rm", db], 60, "docker rm")
|
|
999
|
+
step = "upgrade with the new MariaDB"
|
|
983
1000
|
must([DOCKER, "run", "-d", "--name", db, *database, "-e", "MARIADB_AUTO_UPGRADE=1", images["mysql"]], 120, "docker run of the new MariaDB")
|
|
984
1001
|
wait_for_server(db, 900)
|
|
985
1002
|
upgraded = fingerprint(db)
|
|
986
1003
|
version = maria(db, "SELECT VERSION()")[0][0]
|
|
987
1004
|
checks.append({"check": "upgrade kept every row and ledger", "ok": upgraded == restored,
|
|
988
1005
|
"observed": {"server_version": version, **differences(restored, upgraded)}})
|
|
1006
|
+
step = "migrations, server, Redis, and proxy"
|
|
989
1007
|
code = run([DOCKER, "run", "--name", migration, "--network", name, *app_env, images["migration"]], 1800).returncode
|
|
990
1008
|
checks.append({"check": "migrations exit 0", "ok": code == 0, "observed": code if code == 0 else {"exit": code, "errors": error_names(migration)}})
|
|
991
1009
|
migrated = fingerprint(db)
|
|
@@ -1004,7 +1022,7 @@ def op_bigcapital_rehearse(args):
|
|
|
1004
1022
|
checks.append({"check": "proxy configuration valid", "ok": code == 0, "observed": code})
|
|
1005
1023
|
outcome["fingerprint"] = summary(migrated)
|
|
1006
1024
|
except Failed as error:
|
|
1007
|
-
checks.append({"check": "completed", "ok": False, "observed": str(error)})
|
|
1025
|
+
checks.append({"check": "completed", "ok": False, "observed": {"step": step, "error": str(error), "database_errors": server_errors(db)}})
|
|
1008
1026
|
finally:
|
|
1009
1027
|
for container in (server, redis, migration, db):
|
|
1010
1028
|
run([DOCKER, "rm", "-f", "-v", container], 120)
|