@astrofoundry/pi-astro 0.40.49 → 0.40.51

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -18,6 +18,7 @@ import type { InvestigationRequest } from "./security-monitor.ts";
18
18
  export interface WatchRest {
19
19
  createMessage(channelId: string, message: CreateMessage): Promise<DiscordMessage>;
20
20
  startThread(channelId: string, messageId: string, name: string): Promise<{ id: string }>;
21
+ archiveThread(threadId: string): Promise<void>;
21
22
  }
22
23
 
23
24
  export interface WatchDeps {
@@ -129,7 +130,8 @@ export class ArcaneWatch {
129
130
  const starter = await this.deps.rest.createMessage(thread.id, { content: "🔎 The arcane specialist reads only; a risky read still asks an owner here. Write in this thread to continue." });
130
131
  const owner = this.deps.config().owners[0];
131
132
  const ok = await this.deps.investigate({ threadId: thread.id, starterId: starter.id, userId: owner, task });
132
- if (!ok) this.deps.log(`arcane watch: the investigation of ${name} did not answer`);
133
+ if (ok) await this.deps.rest.archiveThread(thread.id).catch((err) => this.deps.log(`arcane watch: archiving the thread of ${name} failed: ${reason(err)}`));
134
+ else this.deps.log(`arcane watch: the investigation of ${name} did not answer`);
133
135
  } catch (err) {
134
136
  this.deps.log(`arcane watch: investigation of ${name} failed: ${reason(err)}`);
135
137
  await this.post(channelId, { content: `❌ Could not start the investigation of ${name} (${reason(err)}); ask the arcane specialist in the channel.`, replyTo: messageId });
@@ -199,6 +199,7 @@ describe("container watch monitor", () => {
199
199
  function harness() {
200
200
  const messages: { channelId: string; message: CreateMessage }[] = [];
201
201
  const threads: string[] = [];
202
+ const archived: string[] = [];
202
203
  const investigations: InvestigationRequest[] = [];
203
204
  const logs: string[] = [];
204
205
  const clock = { now: T0 };
@@ -214,6 +215,9 @@ describe("container watch monitor", () => {
214
215
  threads.push(messageId);
215
216
  return { id: `t${messageId}` };
216
217
  },
218
+ archiveThread: async (threadId) => {
219
+ archived.push(threadId);
220
+ },
217
221
  },
218
222
  runEntry: async (service, caller, args) => {
219
223
  expect([service, caller, ...args]).toEqual(["arcane", "arcane-watch", "container-watch"]);
@@ -229,7 +233,7 @@ describe("container watch monitor", () => {
229
233
  stopping: () => false,
230
234
  now: () => clock.now,
231
235
  });
232
- return { watch, messages, threads, investigations, logs, clock, answer };
236
+ return { watch, messages, threads, archived, investigations, logs, clock, answer };
233
237
  }
234
238
 
235
239
  const SCHEDULE = config().reports[0];
@@ -250,6 +254,7 @@ describe("container watch monitor", () => {
250
254
  expect(h.investigations).toHaveLength(1);
251
255
  expect(h.investigations[0]).toMatchObject({ threadId: "t500", starterId: "501", userId: OWNER });
252
256
  expect(h.investigations[0].task).toContain("restarted 3 times");
257
+ expect(h.archived).toEqual(["t500"]);
253
258
  });
254
259
 
255
260
  it("posts a host-wide restart once, untagged, and investigates nothing", async () => {
@@ -57,7 +57,7 @@ export function edgeStatus(status: string | null, updates: string | null): AreaS
57
57
  const parts = [
58
58
  down.length === 0 ? `${units.length} services active` : `down: ${down.join(", ")}`,
59
59
  `WireGuard ${handshake || "unknown"}`,
60
- rows.get("reboot required") === "yes" ? "reboot pending (it reboots itself at 02:30 UTC)" : "no reboot pending",
60
+ rows.get("reboot required") === "yes" ? "reboot pending (it reboots itself at 01:20 UTC)" : "no reboot pending",
61
61
  upgradable === null ? "updates unknown" : `${upgradable} upgradable package${upgradable === 1 ? "" : "s"}`,
62
62
  ];
63
63
  const tunnel = age === null || age > STALE_HANDSHAKE_S;
@@ -171,7 +171,7 @@ export function guestOwner(name: string): string {
171
171
  return "Proxmox";
172
172
  }
173
173
 
174
- /** Each guest's and Synapse's `updates` after the 04:30 run: anything still waiting did not install and needs a look. */
174
+ /** Each guest's and Synapse's `updates` after the 02:25 run: anything still waiting did not install and needs a look. */
175
175
  export function updatesStatus(hosts: Readonly<Record<string, string | null>>): AreaStatus {
176
176
  const names = Object.keys(hosts);
177
177
  const wazuh = wazuhRelease(hosts);
@@ -319,7 +319,7 @@ export function mediaStatus(sab: unknown): AreaStatus {
319
319
  return { area: "Media", line, ...(warnings > 0 ? { problem: `Media: the downloader shows ${warnings} warning${warnings === 1 ? "" : "s"}`, route: "media" as const } : {}) };
320
320
  }
321
321
 
322
- /** Every guest is backed up nightly at 02:30; an archive older than this means last night's run missed it. */
322
+ /** Every guest is backed up nightly from 02:00; an archive older than this means last night's run missed it. */
323
323
  const MISSED_BACKUP_S = 30 * HOUR;
324
324
 
325
325
  /** `backup vzdump-archives` and `restic-check-status`: last night's Proxmox archives and the weekly restic check. */
@@ -302,6 +302,10 @@ describe("rest", () => {
302
302
  await rest.startThread("c1", "m9", "Rule 100021 on arcane");
303
303
  expect(calls[3].url).toBe("https://discord.com/api/v10/channels/c1/messages/m9/threads");
304
304
  expect(JSON.parse(calls[3].init.body as string)).toEqual({ name: "Rule 100021 on arcane", auto_archive_duration: 10080 });
305
+ await rest.archiveThread("m9");
306
+ expect(calls[4].url).toBe("https://discord.com/api/v10/channels/m9");
307
+ expect(calls[4].init.method).toBe("PATCH");
308
+ expect(JSON.parse(calls[4].init.body as string)).toEqual({ archived: true });
305
309
  });
306
310
  });
307
311
 
@@ -843,7 +843,7 @@ class Bridge {
843
843
  const posted = await this.rest.createMessage(report.channelId, { content: autoFailureText(a, problems, rollback, owners), mentionUsers: owners, ...(page ? { replyTo: page.id } : {}) });
844
844
  const thread = await this.rest.startThread(report.channelId, posted.id, `Update of ${a.stack}`.slice(0, 100));
845
845
  const starter = await this.rest.createMessage(thread.id, { content: "🔎 The foreman reads only and asks the specialists for evidence. Write here to continue with the arcane specialist." });
846
- void this.investigateIn("foreman", { threadId: thread.id, starterId: starter.id, userId: owners[0], task: foremanTask(a, problems, rollback, Date.now()) });
846
+ void this.investigateIn("foreman", { threadId: thread.id, starterId: starter.id, userId: owners[0], task: foremanTask(a, problems, rollback, Date.now()) }).then((ok) => (ok ? this.archiveThread(thread.id) : undefined));
847
847
  } catch (err) {
848
848
  log(`arcane auto: reporting ${a.stack} failed: ${err instanceof Error ? err.message : String(err)}`);
849
849
  }
@@ -1192,6 +1192,11 @@ class Bridge {
1192
1192
  }
1193
1193
  }
1194
1194
 
1195
+ /** A thread whose work is done leaves the channel list; a failure only costs the 7-day automatic archive. */
1196
+ private async archiveThread(threadId: string): Promise<void> {
1197
+ await this.rest.archiveThread(threadId).catch((err) => log(`discord: archiving thread ${threadId} failed: ${err instanceof Error ? err.message : String(err)}`));
1198
+ }
1199
+
1195
1200
  private rememberCarefulThread(report: ArcaneReport, stack: string, threadId: string): void {
1196
1201
  const current = readPostedArcane();
1197
1202
  const target = current && current.date === report.date ? current : report;
@@ -1214,6 +1219,7 @@ class Bridge {
1214
1219
  const outcome = stackOutcome(ran.answer);
1215
1220
  await this.markArcane(report, entry.stack, outcome.state, outcome.detail);
1216
1221
  log(`arcane: careful ${entry.stack} ${outcome.detail}`);
1222
+ await this.archiveThread(threadId);
1217
1223
  if (outcome.state !== "updated") return;
1218
1224
  const applied: Applied = { stack: entry.stack, finishedAt: ran.at, detail: outcome.detail, ...(outcome.commit ? { commit: outcome.commit } : {}) };
1219
1225
  void this.superviseArcane(report, [applied]).catch((err) => log(`arcane: supervising ${entry.stack} failed: ${err instanceof Error ? err.message : String(err)}`));
@@ -113,6 +113,11 @@ export class DiscordRest {
113
113
  return (await this.call("POST", `/channels/${channelId}/messages/${messageId}/threads`, JSON.stringify({ name: name.slice(0, 100), auto_archive_duration: 10080 }), "application/json")).json() as Promise<{ id: string }>;
114
114
  }
115
115
 
116
+ /** Takes a thread the bot started out of the channel list once its work is done; it stays readable, and any new message reopens it. */
117
+ async archiveThread(threadId: string): Promise<void> {
118
+ await this.call("PATCH", `/channels/${threadId}`, JSON.stringify({ archived: true }), "application/json");
119
+ }
120
+
116
121
  /** Shows the "Cortex is typing…" indicator for about ten seconds. */
117
122
  async triggerTyping(channelId: string): Promise<void> {
118
123
  await this.call("POST", `/channels/${channelId}/typing`);
@@ -32,6 +32,7 @@ class FakeRest implements SecurityRest {
32
32
  replies: string[] = [];
33
33
  deferred = 0;
34
34
  threads: string[] = [];
35
+ archived: string[] = [];
35
36
  failComponents = false;
36
37
  private next = 100;
37
38
 
@@ -57,6 +58,9 @@ class FakeRest implements SecurityRest {
57
58
  this.threads.push(messageId);
58
59
  return { id: messageId };
59
60
  }
61
+ async archiveThread(threadId: string) {
62
+ this.archived.push(threadId);
63
+ }
60
64
  }
61
65
 
62
66
  function wazuh(over: Partial<WazuhEvent>): WazuhEvent {
@@ -257,6 +261,7 @@ describe("security monitor", () => {
257
261
  expect(h.investigations[0].task).toContain("Read only: change nothing.");
258
262
  expect(h.rest.messages[0].channelId).toBe(messageId);
259
263
  expect(readSecurityState().alerts[0]).toMatchObject({ investigation: "done", threadId: messageId });
264
+ expect(h.rest.archived).toEqual([messageId]);
260
265
  expect(h.rest.edits.at(-1)?.components[1].components?.[0].label).toBe("Investigate again");
261
266
  await h.monitor.onButton({ interactionId: "i", token: "t", userId: OWNER, messageId, customId: "security:investigate" });
262
267
  expect(h.rest.threads).toHaveLength(1);
@@ -61,6 +61,7 @@ export interface SecurityRest {
61
61
  ephemeralReply(interactionId: string, interactionToken: string, content: string): Promise<void>;
62
62
  deferUpdate(interactionId: string, interactionToken: string): Promise<void>;
63
63
  startThread(channelId: string, messageId: string, name: string): Promise<{ id: string }>;
64
+ archiveThread(threadId: string): Promise<void>;
64
65
  }
65
66
 
66
67
  export interface InvestigationRequest {
@@ -616,6 +617,7 @@ export class SecurityMonitor {
616
617
  this.deps.log(`security: investigation of ${alert.key} failed: ${reason(err)}`);
617
618
  }
618
619
  await this.setInvestigation(alert.messageId, ok ? "done" : "failed", threadId);
620
+ if (ok) await this.deps.rest.archiveThread(threadId).catch((err) => this.deps.log(`security: archiving the thread of ${alert.key} failed: ${reason(err)}`));
619
621
  }
620
622
 
621
623
  /** An investigation a restart cut off can be started again. */
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@astrofoundry/pi-astro",
3
- "version": "0.40.49",
3
+ "version": "0.40.51",
4
4
  "description": "Personal pi customizations (extensions, subagents, skills, prompts, themes) for the pi coding agent.",
5
5
  "keywords": [
6
6
  "pi-package"
@@ -7,12 +7,12 @@ description: How the backup specialist checks and runs the homelab backup layers
7
7
 
8
8
  Four layers protect the homelab. Every command is a fixed operation on Pulsar (root) or on the Arcane VM (user `arcane`); `["--help"]` lists them.
9
9
 
10
- | Layer | What | Schedule | Commands |
10
+ | Layer | What | Schedule (Europe/Madrid) | Commands |
11
11
  |---|---|---|---|
12
- | 1 | vzdump of every guest to PVE storage `unas-backup` (NAS `//10.0.50.50/Shared_Drive/BACKUP/pulsar/dump`), snapshot mode, zstd, 7 daily + 4 weekly | 02:30 | `vzdump-jobs`, `vzdump-archives`, `vzdump-tasks`, `vzdump-run <vmid>` |
13
- | 2 | App-consistent service backups from the Arcane VM into the shared Restic repository (`/mnt/backup/restic` there), one tag per service, 30 daily + 12 monthly | 03:00, check Sunday 04:00 | `arcane-backup-status|run|journal`, `restic-check-status|run|journal` |
14
- | 3 | Pulsar host config, Technitium ZIP exports, Identity `pg_dump` into Restic, tag `pulsar-host` | 03:30 | `host-backup-status|run|journal` |
15
- | 4 | Offsite copy of the Restic repository to `gs://ardilla-backup/pulsar/restic` from LXC 102 (append-only service account) | 04:30 UTC | `gcs-status|run|journal` |
12
+ | 1 | vzdump of every guest to PVE storage `unas-backup` (NAS `//10.0.50.50/Shared_Drive/BACKUP/pulsar/dump`), snapshot mode, zstd, 7 daily + 4 weekly | 02:00 | `vzdump-jobs`, `vzdump-archives`, `vzdump-tasks`, `vzdump-run <vmid>` |
13
+ | 2 | App-consistent service backups from the Arcane VM into the shared Restic repository (`/mnt/backup/restic` there), one tag per service, 30 daily + 12 monthly | 02:15, check Sunday 02:25 | `arcane-backup-status|run|journal`, `restic-check-status|run|journal` |
14
+ | 3 | Pulsar host config, Technitium ZIP exports, Identity `pg_dump` into Restic, tag `pulsar-host` | 02:20 | `host-backup-status|run|journal` |
15
+ | 4 | Offsite copy of the Restic repository to `gs://ardilla-backup/pulsar/restic` from LXC 102 (append-only service account) | 03:10 | `gcs-status|run|journal` |
16
16
 
17
17
  Restic commands run on Pulsar against `/mnt/pve/unas-backup/restic` with the host's password file: `restic-snapshots [tag]`, `restic-stats [tag]`, `restic-ls <snapshot> [path]`, `restic-restore <snapshot> <path>`, `restore-ls`, `restore-clean`. `restic-restore` writes under `/var/tmp/spc-restore/<snapshot>/` on Pulsar and nowhere else.
18
18
 
@@ -45,7 +45,7 @@ A service's backup is confirmed when `arcane-backup-status` shows `Result=succes
45
45
 
46
46
  ## Mira guests
47
47
 
48
- SPACE-X (Mira) writes its own vzdump archives to TESLA (`tesla-backup`, `BACKUP/space-x/dump`), and Pulsar's `mira-backup-pull.timer` copies them to the UNAS (`BACKUP/space-x/dump`) at 05:00 without mirroring Mira's deletions. This tool does not read SPACE-X; the proxmox specialist's `space-x tasks` shows the runs, and Kuma monitors `space-x-backup` and `mira-backup-pull` carry the heartbeats.
48
+ SPACE-X (Mira) writes its own vzdump archives to TESLA (`tesla-backup`, `BACKUP/space-x/dump`), and Pulsar's `mira-backup-pull.timer` copies them to the UNAS (`BACKUP/space-x/dump`) at 03:05 without mirroring Mira's deletions. This tool does not read SPACE-X; the proxmox specialist's `space-x tasks` shows the runs, and Kuma monitors `space-x-backup` and `mira-backup-pull` carry the heartbeats.
49
49
 
50
50
  ## Reading the output
51
51
 
@@ -48,7 +48,7 @@ Removing a hostname is the same with a removed line; check first that no public
48
48
 
49
49
  ## Vulnerability on the VPS
50
50
 
51
- Wazuh reports a CVE against a package (the security specialist passes `package`, installed `version`, and `condition`, the fixed version). The VPS patches itself: unattended-upgrades installs Debian, Debian-Security, and Google guest-agent updates once a day and reboots at 02:30 UTC when a package needs it. It does not take `trixie-updates` or Google's cloud-sdk repository, so packages from those (such as `tzdata` and `google-cloud-cli`) wait for the operator. Answer which of three states applies:
51
+ Wazuh reports a CVE against a package (the security specialist passes `package`, installed `version`, and `condition`, the fixed version). The VPS patches itself: unattended-upgrades installs Debian, Debian-Security, `trixie-updates`, Google guest-agent, Google Cloud CLI, and CrowdSec bouncer updates at 02:05 Europe/Madrid, after a package-list refresh at 02:00, and reboots at 01:20 UTC when a package needs it (`00-frontdoor-vps/system/52unattended-upgrades-frontdoor`). Answer which of three states applies:
52
52
 
53
53
  1. `package <name>`, then `changelog <name>`: if the installed version comes from the security origin and its changelog entry names the CVE, it is patched; say by which upload and, from `auto-upgrades` or the dpkg log, when. Wazuh may keep flagging the new version until its feed learns the fixed version; that is feed lag, not an open hole.
54
54
  2. Otherwise, if the candidate's changelog names the CVE, it is scheduled: `auto-upgrades` shows when `apt-daily-upgrade.timer` runs next. Say so with the time.
@@ -64,7 +64,7 @@ The `security` tool is read-only. Every call is a fixed command on the guest; `[
64
64
 
65
65
  You observe; the owning specialist acts. When a finding concerns a host in the table, call that specialist through `subagent` with the facts (`agent`, `package`, installed `version`, `condition`, `cve`) and a precise question, then report its answer with your own. Do not tell the caller to run commands on a host that has an owner.
66
66
 
67
- Debian guests and Synapse install every stable update each night (04:30 Europe/Madrid, technitium2 04:50, the Mira guests 04:30 Asia/Dubai) and reboot right after when a package needs it; `wazuh-agent` is held. The VPS installs Debian and Debian-Security updates once a day and reboots at 02:30 UTC when needed; packages from `trixie-updates` wait for the operator. The morning run upgrades the Pulsar and SPACE-X hosts and offers the owner a Reboot button. For a vulnerability alert, ask the owner whether the package is already upgraded, will be at the next run (the fix is in the index), or is stuck (the fix is not in the index or the run failed), and to confirm from the package changelog that the installed version names the CVE. Only the stuck case needs the operator; say so plainly, with the versions. Answer with one of: patched (version, when, by what), scheduled (next run time), stuck (why), or feed lag (patched, Wazuh not updated yet).
67
+ Debian guests and Synapse install every stable update each night (02:25 Europe/Madrid, technitium2 02:30, the Mira guests 02:35, each five minutes after a package-list refresh) and reboot right after when a package needs it; `wazuh-agent` is held. The VPS installs Debian, Debian-Security, `trixie-updates`, Google guest-agent, Google Cloud CLI, and CrowdSec bouncer updates at 02:05 Europe/Madrid and reboots at 01:20 UTC when needed. The morning run upgrades the Pulsar and SPACE-X hosts and offers the owner a Reboot button. For a vulnerability alert, ask the owner whether the package is already upgraded, will be at the next run (the fix is in the index), or is stuck (the fix is not in the index or the run failed), and to confirm from the package changelog that the installed version names the CVE. Only the stuck case needs the operator; say so plainly, with the versions. Answer with one of: patched (version, when, by what), scheduled (next run time), stuck (why), or feed lag (patched, Wazuh not updated yet).
68
68
 
69
69
  ## Operator actions (describe, do not perform)
70
70
 
@@ -792,10 +792,10 @@ def fingerprint(container):
792
792
  union = " UNION ALL ".join(f"SELECT '{s}.{t}', COUNT(*) FROM `{s}`.`{t}`" for s, t in tables)
793
793
  rows = {name: int(count) for name, count in maria(container, union, 1800)}
794
794
  ledgers = {}
795
- for schema in schemas:
796
- if [schema, "accounts_transactions"] in tables:
797
- [[credit, debit]] = maria(container, "SELECT COALESCE(SUM(credit), 0), COALESCE(SUM(debit), 0) "
798
- f"FROM `{schema}`.`accounts_transactions`")
795
+ # Bigcapital's knex mappers upper-case its table names (ACCOUNTS_TRANSACTIONS), but not knex_migrations.
796
+ for schema, table in tables:
797
+ if table.lower() == "accounts_transactions":
798
+ [[credit, debit]] = maria(container, f"SELECT COALESCE(SUM(credit), 0), COALESCE(SUM(debit), 0) FROM `{schema}`.`{table}`")
799
799
  ledgers[schema] = [credit, debit]
800
800
  return {"schemas": schemas, "rows": rows, "ledgers": ledgers}
801
801
 
@@ -947,6 +947,18 @@ def error_names(container):
947
947
  return sorted(set(DIAGNOSTIC.findall(text)))[:20]
948
948
 
949
949
 
950
+ def server_errors(container):
951
+ """MariaDB's and its entrypoint's own error lines, without repeats; they name a failed step, not the books."""
952
+ result = run([DOCKER, "logs", "--tail", "400", container], 60)
953
+ lines = []
954
+ for line in (result.stdout + result.stderr).decode("utf-8", "replace").splitlines():
955
+ found = re.search(r"\[ERROR\].*|ERROR [0-9]{4} \([0-9A-Z]{5}\).*", line)
956
+ text = found.group(0)[:200] if found else None
957
+ if text and text not in lines:
958
+ lines.append(text)
959
+ return lines[-6:]
960
+
961
+
950
962
  def op_bigcapital_rehearse(args):
951
963
  """Restores a backup into the current MariaDB on an internal network, upgrades it with the build's MariaDB,
952
964
  runs the build's migrations, starts its server and Redis, and validates its proxy configuration; then
@@ -966,26 +978,32 @@ def op_bigcapital_rehearse(args):
966
978
  name = f"bigcapital-rehearsal-{build}"
967
979
  db, migration, redis, server = (f"{name}-{part}" for part in ("db", "migration", "redis", "server"))
968
980
  root_password, user_password, jwt_secret = (secrets.token_hex(24) for _ in range(3))
969
- database = ["--network", name, "--network-alias", "mysql", "-v", f"{name}:/var/lib/mysql", "-e", f"MYSQL_ROOT_PASSWORD={root_password}"]
981
+ # The stack's database settings: the image's own MYSQL_DATABASE default names no database, and the
982
+ # upgrade's healthcheck-user step fails on it.
983
+ database = ["--network", name, "--network-alias", "mysql", "-v", f"{name}:/var/lib/mysql", "-e", f"MYSQL_ROOT_PASSWORD={root_password}",
984
+ "-e", f"MYSQL_DATABASE={BIGCAPITAL_SYSTEM_DB}", "-e", "MYSQL_USER=bigcapital", "-e", f"MYSQL_PASSWORD={user_password}"]
970
985
  app_env = [part for key, value in {**BIGCAPITAL_SERVER_ENV, "DB_PASSWORD": user_password}.items() for part in ("-e", f"{key}={value}")]
971
986
  checks = []
972
987
  outcome = {"stamp": args[0], "build": build, "images": images, "current_mysql_image": current, "checks": checks}
988
+ step = "network and volume"
973
989
  try:
974
990
  must([DOCKER, "network", "create", "--internal", name], 60, "docker network create")
975
991
  must([DOCKER, "volume", "create", name], 60, "docker volume create")
976
- must([DOCKER, "run", "-d", "--name", db, *database, "-e", f"MYSQL_DATABASE={BIGCAPITAL_SYSTEM_DB}", "-e", "MYSQL_USER=bigcapital",
977
- "-e", f"MYSQL_PASSWORD={user_password}", current], 120, "docker run of the current MariaDB")
992
+ step = "restore into the current MariaDB"
993
+ must([DOCKER, "run", "-d", "--name", db, *database, current], 120, "docker run of the current MariaDB")
978
994
  wait_for_server(db, 300)
979
995
  restore_dump(db, source)
980
996
  restored = fingerprint(db)
981
997
  must([DOCKER, "stop", "-t", "300", db], 330, "docker stop of the current MariaDB")
982
998
  must([DOCKER, "rm", db], 60, "docker rm")
999
+ step = "upgrade with the new MariaDB"
983
1000
  must([DOCKER, "run", "-d", "--name", db, *database, "-e", "MARIADB_AUTO_UPGRADE=1", images["mysql"]], 120, "docker run of the new MariaDB")
984
1001
  wait_for_server(db, 900)
985
1002
  upgraded = fingerprint(db)
986
1003
  version = maria(db, "SELECT VERSION()")[0][0]
987
1004
  checks.append({"check": "upgrade kept every row and ledger", "ok": upgraded == restored,
988
1005
  "observed": {"server_version": version, **differences(restored, upgraded)}})
1006
+ step = "migrations, server, Redis, and proxy"
989
1007
  code = run([DOCKER, "run", "--name", migration, "--network", name, *app_env, images["migration"]], 1800).returncode
990
1008
  checks.append({"check": "migrations exit 0", "ok": code == 0, "observed": code if code == 0 else {"exit": code, "errors": error_names(migration)}})
991
1009
  migrated = fingerprint(db)
@@ -1004,7 +1022,7 @@ def op_bigcapital_rehearse(args):
1004
1022
  checks.append({"check": "proxy configuration valid", "ok": code == 0, "observed": code})
1005
1023
  outcome["fingerprint"] = summary(migrated)
1006
1024
  except Failed as error:
1007
- checks.append({"check": "completed", "ok": False, "observed": str(error)})
1025
+ checks.append({"check": "completed", "ok": False, "observed": {"step": step, "error": str(error), "database_errors": server_errors(db)}})
1008
1026
  finally:
1009
1027
  for container in (server, redis, migration, db):
1010
1028
  run([DOCKER, "rm", "-f", "-v", container], 120)