@softure-ai/backup 0.0.0-stage → 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (107) hide show
  1. package/CHANGELOG.md +18 -0
  2. package/LICENSE +21 -0
  3. package/README.md +223 -2
  4. package/dist/cli/main.d.ts +3 -0
  5. package/dist/cli/main.d.ts.map +1 -0
  6. package/dist/cli/main.js +16 -0
  7. package/dist/cli/main.js.map +1 -0
  8. package/dist/cli/run.d.ts +12 -0
  9. package/dist/cli/run.d.ts.map +1 -0
  10. package/dist/cli/run.js +335 -0
  11. package/dist/cli/run.js.map +1 -0
  12. package/dist/contract.d.ts +41 -0
  13. package/dist/contract.d.ts.map +1 -0
  14. package/dist/contract.js +14 -0
  15. package/dist/contract.js.map +1 -0
  16. package/dist/index.d.ts +21 -0
  17. package/dist/index.d.ts.map +1 -0
  18. package/dist/index.js +30 -0
  19. package/dist/index.js.map +1 -0
  20. package/dist/messages/en.d.ts +16 -0
  21. package/dist/messages/en.d.ts.map +1 -0
  22. package/dist/messages/en.js +16 -0
  23. package/dist/messages/en.js.map +1 -0
  24. package/dist/messages/index.d.ts +21 -0
  25. package/dist/messages/index.d.ts.map +1 -0
  26. package/dist/messages/index.js +4 -0
  27. package/dist/messages/index.js.map +1 -0
  28. package/dist/offsite/config.d.ts +43 -0
  29. package/dist/offsite/config.d.ts.map +1 -0
  30. package/dist/offsite/config.js +130 -0
  31. package/dist/offsite/config.js.map +1 -0
  32. package/dist/offsite/crypto.d.ts +22 -0
  33. package/dist/offsite/crypto.d.ts.map +1 -0
  34. package/dist/offsite/crypto.js +48 -0
  35. package/dist/offsite/crypto.js.map +1 -0
  36. package/dist/offsite/index.d.ts +11 -0
  37. package/dist/offsite/index.d.ts.map +1 -0
  38. package/dist/offsite/index.js +13 -0
  39. package/dist/offsite/index.js.map +1 -0
  40. package/dist/offsite/lifecycle.d.ts +17 -0
  41. package/dist/offsite/lifecycle.d.ts.map +1 -0
  42. package/dist/offsite/lifecycle.js +45 -0
  43. package/dist/offsite/lifecycle.js.map +1 -0
  44. package/dist/offsite/objects.d.ts +19 -0
  45. package/dist/offsite/objects.d.ts.map +1 -0
  46. package/dist/offsite/objects.js +44 -0
  47. package/dist/offsite/objects.js.map +1 -0
  48. package/dist/offsite/postgres.d.ts +32 -0
  49. package/dist/offsite/postgres.d.ts.map +1 -0
  50. package/dist/offsite/postgres.js +97 -0
  51. package/dist/offsite/postgres.js.map +1 -0
  52. package/dist/offsite/restore.d.ts +54 -0
  53. package/dist/offsite/restore.d.ts.map +1 -0
  54. package/dist/offsite/restore.js +67 -0
  55. package/dist/offsite/restore.js.map +1 -0
  56. package/dist/offsite/s3.d.ts +34 -0
  57. package/dist/offsite/s3.d.ts.map +1 -0
  58. package/dist/offsite/s3.js +119 -0
  59. package/dist/offsite/s3.js.map +1 -0
  60. package/dist/offsite/sigv4.d.ts +19 -0
  61. package/dist/offsite/sigv4.d.ts.map +1 -0
  62. package/dist/offsite/sigv4.js +69 -0
  63. package/dist/offsite/sigv4.js.map +1 -0
  64. package/dist/offsite/status-sql.d.ts +10 -0
  65. package/dist/offsite/status-sql.d.ts.map +1 -0
  66. package/dist/offsite/status-sql.js +37 -0
  67. package/dist/offsite/status-sql.js.map +1 -0
  68. package/dist/offsite/upload.d.ts +33 -0
  69. package/dist/offsite/upload.d.ts.map +1 -0
  70. package/dist/offsite/upload.js +98 -0
  71. package/dist/offsite/upload.js.map +1 -0
  72. package/dist/offsite/verify.d.ts +32 -0
  73. package/dist/offsite/verify.d.ts.map +1 -0
  74. package/dist/offsite/verify.js +75 -0
  75. package/dist/offsite/verify.js.map +1 -0
  76. package/dist/schema.d.ts +150 -0
  77. package/dist/schema.d.ts.map +1 -0
  78. package/dist/schema.js +15 -0
  79. package/dist/schema.js.map +1 -0
  80. package/dist/server/index.d.ts +28 -0
  81. package/dist/server/index.d.ts.map +1 -0
  82. package/dist/server/index.js +98 -0
  83. package/dist/server/index.js.map +1 -0
  84. package/examples/backup-verify.yml +84 -0
  85. package/migrations/0001_create_runs.sql +21 -0
  86. package/module.json +12 -0
  87. package/package.json +67 -4
  88. package/src/cli/main.ts +16 -0
  89. package/src/cli/run.ts +338 -0
  90. package/src/contract.ts +46 -0
  91. package/src/index.ts +41 -0
  92. package/src/messages/en.ts +15 -0
  93. package/src/messages/index.ts +6 -0
  94. package/src/offsite/config.ts +157 -0
  95. package/src/offsite/crypto.ts +53 -0
  96. package/src/offsite/index.ts +31 -0
  97. package/src/offsite/lifecycle.ts +47 -0
  98. package/src/offsite/objects.ts +54 -0
  99. package/src/offsite/postgres.ts +115 -0
  100. package/src/offsite/restore.ts +108 -0
  101. package/src/offsite/s3.ts +144 -0
  102. package/src/offsite/sigv4.ts +84 -0
  103. package/src/offsite/status-sql.ts +45 -0
  104. package/src/offsite/upload.ts +116 -0
  105. package/src/offsite/verify.ts +97 -0
  106. package/src/schema.ts +17 -0
  107. package/src/server/index.ts +111 -0
package/CHANGELOG.md ADDED
@@ -0,0 +1,18 @@
1
+ # Changelog
2
+
3
+ Newest first. Each version lists what changed for an app that uses the module. When an app has run a version in
4
+ production, the version gets a line `verified in: <app>@<commit>` ([docs/05](../../docs/05-adoption-playbook.md),
5
+ "Definition of done").
6
+
7
+ ## 0.1.0
8
+
9
+ - **First version (#1210):** encrypted off-site copies of the nightly backup of `@softure-ai/deploy`. `softure-backup
10
+ upload` encrypts the night's dump (and data archive) with `age` to `BACKUP_AGE_RECIPIENT` and PUTs it to any
11
+ S3-compatible store (R2, B2, Hetzner, MinIO, AWS) under `<prefix>/<file>.age` with a write-only token; it never
12
+ lists, reads or deletes, and the bucket's lifecycle rule (`softure-backup lifecycle`) expires old copies. Each run
13
+ is recorded in `backup.runs` (the module's migration): time, result (`ok`, `failed`, `not_configured`), objects
14
+ with size and sha256, a safe error code and message. The app reads it with `getLastBackup()` and
15
+ `listBackupRuns()` and exports `registerBackupMetrics()` gauges (`softure.backup.last_success.age` for an alert
16
+ above 26 h). `softure-backup restore` restores into a new database unless `--overwrite --into=<db>`;
17
+ `softure-backup verify` restores the newest copy into a throwaway database, runs the deploy's schema guard and row
18
+ counts and drops it; `fetch` decrypts a data archive. The private key comes from a file or stdin only.
package/LICENSE ADDED
@@ -0,0 +1,21 @@
1
+ MIT License
2
+
3
+ Copyright (c) 2026 SOFTURE
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in all
13
+ copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21
+ SOFTWARE.
package/README.md CHANGED
@@ -1,3 +1,224 @@
1
- # Temporary Holding Version
1
+ # @softure-ai/backup
2
2
 
3
- This version is a temporary placeholder for this package. An operational version to replace this has been submitted for review and is awaiting a staged release.
3
+ Encrypted off-site copies of the nightly backup of [`@softure-ai/deploy`](../../tools/deploy/README.md), their status
4
+ for the app, and a restore and a weekly verify that need a private key the server never holds.
5
+
6
+ ## 1. What it provides
7
+
8
+ - **`softure-backup upload`**, run by `deploy.sh maintain` right after the night's `pg_dump` (step `offsite-backup`):
9
+ it encrypts the new dump (and the data archive, when the app has `data.backupPaths`) with
10
+ [age](https://age-encryption.org) to a **public key** and PUTs it to any S3-compatible store (AWS S3, Cloudflare R2,
11
+ Backblaze B2, Hetzner Object Storage, MinIO, SeaweedFS…) as `<prefix>/<file>.age`, with a **write-only token**. It
12
+ never lists, reads or deletes anything remote: the bucket's lifecycle rule expires old copies. No second `pg_dump`,
13
+ no extra compose service: it reuses the dump `maintain` already took.
14
+ - **A status row per run** in `backup.runs`: start and end time, result (`ok`, `failed`, `not_configured`), each
15
+ object's key, size and sha256, and on failure a safe error code and message (no secret, no URL).
16
+ - **For the app:** `getLastBackup(db)` (the newest run and the newest successful one, for a settings screen),
17
+ `listBackupRuns(db, { limit })`, `getLastSuccessAgeSeconds(status, now)`, `registerBackupMetrics({ db })`
18
+ (OpenTelemetry gauges through `@softure-ai/observability`, among them the age of the newest good copy for an alert
19
+ above 26 hours) and a health check of the table.
20
+ - **For the operator:** `softure-backup restore` (into a **new** database by default; over an existing one only with
21
+ `--into=<db> --overwrite`), `softure-backup fetch` (decrypt one object, e.g. the data archive), `softure-backup
22
+ verify` (newest copy → throwaway database → the deploy's schema guard and row counts → drop; meant to run weekly,
23
+ see [examples/backup-verify.yml](examples/backup-verify.yml)) and `softure-backup lifecycle` (the bucket rule).
24
+ - The private key is read from a file or stdin only. An argument that holds `AGE-SECRET-KEY-` is refused, so the key
25
+ never shows in `ps` or a shell history.
26
+
27
+ ## 2. Installation
28
+
29
+ ```bash
30
+ npm install @softure-ai/backup
31
+ ```
32
+
33
+ Then add the module to `softure.config.ts` (section 3) and run `softure migrate`: the table is what the nightly run
34
+ writes its status into. The server does not need the package installed: `deploy.sh` runs the CLI with `npx` (or in
35
+ its helper tools image on a host without Node), at the version the deploy release pins.
36
+
37
+ ## 3. Configuration
38
+
39
+ ```ts
40
+ // softure.config.ts
41
+ import { backup } from "@softure-ai/backup";
42
+
43
+ export default defineConfig({
44
+ modules: [/* … */, backup()],
45
+ });
46
+ ```
47
+
48
+ There are no options: everything the off-site copy needs comes from the environment (section 6), so the code is the
49
+ same with and without it. An app that never sets `BACKUP_S3_BUCKET` sees `not_configured` and nothing fails.
50
+
51
+ **The deploy side.** `softure-deploy init` (0.7.0 and later) writes into the compose file of an app with a database an
52
+ `x-offsite-backup` block of optional names, which makes `env render` put them into `.env.prod`:
53
+
54
+ ```yaml
55
+ x-offsite-backup:
56
+ BACKUP_S3_ENDPOINT: ${BACKUP_S3_ENDPOINT:-}
57
+ BACKUP_S3_REGION: ${BACKUP_S3_REGION:-}
58
+ BACKUP_S3_BUCKET: ${BACKUP_S3_BUCKET:-}
59
+ BACKUP_S3_PREFIX: ${BACKUP_S3_PREFIX:-}
60
+ BACKUP_S3_ACCESS_KEY_ID: ${BACKUP_S3_ACCESS_KEY_ID:-}
61
+ BACKUP_S3_SECRET_ACCESS_KEY: ${BACKUP_S3_SECRET_ACCESS_KEY:-}
62
+ BACKUP_AGE_RECIPIENT: ${BACKUP_AGE_RECIPIENT:-}
63
+ ```
64
+
65
+ No container gets these values: only `deploy.sh maintain` reads them on the host and hands them to the CLI's
66
+ environment. `deploy.json` needs nothing new; `database.rowCountTables` and `database.excludeTableData` are what
67
+ `verify --config=deploy.json` counts.
68
+
69
+ ## 4. Mounting
70
+
71
+ Nothing to mount: no route or page. A settings screen reads the status on the server:
72
+
73
+ ```ts
74
+ import { getLastBackup, getLastSuccessAgeSeconds } from "@softure-ai/backup/server";
75
+
76
+ const status = await getLastBackup(db); // { result, lastRun, lastSuccess }
77
+ const ageSeconds = getLastSuccessAgeSeconds(status, new Date()); // null until a run succeeded
78
+ ```
79
+
80
+ Metrics, once at start-up next to `startObservability()`: `registerBackupMetrics({ db })`. The `@softure-ai/ops`
81
+ health route picks up the module's health check (the table exists and answers).
82
+
83
+ ## 5. Migrations and tables
84
+
85
+ Schema `backup`, migration `0001_create_runs.sql`, table `backup.runs`: `id`, `started_at`, `finished_at`, `result`
86
+ (`ok`, `failed`, `not_configured`), `objects` (jsonb: kind `database` or `data`, key, bytes, sha256), `size_bytes`,
87
+ `error_code`, `error_message`. A constraint keeps an error exactly on failed runs. The host writes the rows through
88
+ `psql` as the Postgres superuser (`POSTGRES_USER`) and prunes rows older than 400 days in the same transaction; the
89
+ app only reads them.
90
+
91
+ ## 6. Environment variables
92
+
93
+ On the server (`.env.prod`, from the deploy job), all optional; `BACKUP_S3_BUCKET` switches the off-site copy on:
94
+
95
+ | Name | What it is |
96
+ | --- | --- |
97
+ | `BACKUP_S3_ENDPOINT` | The store's S3 endpoint, `https://…` (`http://` only for a local store). |
98
+ | `BACKUP_S3_REGION` | The signing region; default `auto` (R2). AWS needs the bucket's region. |
99
+ | `BACKUP_S3_BUCKET` | The bucket. |
100
+ | `BACKUP_S3_PREFIX` | The folder in the bucket; default the app's name. One prefix per app, so apps can share a bucket. |
101
+ | `BACKUP_S3_ACCESS_KEY_ID`, `BACKUP_S3_SECRET_ACCESS_KEY` | The **write-only** token (secret). |
102
+ | `BACKUP_AGE_RECIPIENT` | The age **public** key, `age1…`. Not a secret. |
103
+
104
+ On the operator's machine or the verify job, never on the server: the same endpoint, region, bucket and prefix, plus
105
+ `BACKUP_S3_READ_ACCESS_KEY_ID` and `BACKUP_S3_READ_SECRET_ACCESS_KEY` (the **read-only** token),
106
+ `BACKUP_RESTORE_DATABASE_URL` / `BACKUP_VERIFY_DATABASE_URL` (a superuser's `postgres://` URL of the Postgres to
107
+ restore into) and optionally `BACKUP_PG_RESTORE` (the `pg_restore` to run; it must be at least the server's major).
108
+ The private key is not an environment variable of the CLI: pass it with `--identity=<file>` or `--identity=-`.
109
+
110
+ Every name is validated with zod: some names set but not all, a bad endpoint, bucket, prefix or recipient is
111
+ `config_invalid`, listed by name and never by value. Secret values are redacted from every message.
112
+
113
+ ## 7. Switches
114
+
115
+ None. Whether the copy runs is the presence of `BACKUP_S3_BUCKET` on the server.
116
+
117
+ ## 8. Appearance
118
+
119
+ No components; the app renders `getLastBackup()` in its own UI.
120
+
121
+ ## 9. Copy
122
+
123
+ `src/messages/en.ts` (`backupMessages`): a label per result (`result.ok`, `result.failed`, `result.not_configured`)
124
+ and a sentence per error code (`errors.config_invalid`, `file_missing`, `encrypt_failed`, `upload_denied`,
125
+ `upload_failed`, `network_error`) for a settings screen. English only; an app overrides them through its message
126
+ layer.
127
+
128
+ ## 10. Hooks
129
+
130
+ None in the app. On the server the step is part of `maintain`; its line is `step|offsite-backup|ok|<n> object(s), <bytes>
131
+ bytes in <s> s` or `step|offsite-backup|ok|not configured`. A failed upload does not stop the steps after it (test
132
+ restore, cleanup, hooks); `maintain` then ends `result|failed|offsite-backup|<message>`, which the cron's mail or log
133
+ shows. When the app has no `backup.runs` table yet, the copy still runs and the step line ends `; status not recorded
134
+ (add @softure-ai/backup to the app and migrate)`.
135
+
136
+ ## 11. GDPR
137
+
138
+ `backup.runs` holds no personal data. The copies hold the whole database, encrypted, so a person's erased data lives on
139
+ in them until the lifecycle rule expires them: keep the rule's days (default 30) in the privacy policy's retention
140
+ section. The module exports and deletes nothing for a user.
141
+
142
+ ## 12. Limitations
143
+
144
+ - One PUT per file, up to 5 GiB per encrypted object (no multipart upload). A larger database needs a store-side
145
+ replication instead.
146
+ - The write-only property is the token's: the CLI never reads or deletes, but only the provider can make the token
147
+ unable to. Where a provider has no write-only key, turn on its object lock (section "Owner guide", step 2).
148
+ - `verify` needs the private key, so it cannot run in `maintain`: run it weekly from a CI job or the owner's machine.
149
+ The server's own weekly `verify-backup` step keeps checking the local dump.
150
+ - Retention is the bucket's lifecycle rule only; nothing here deletes a remote object.
151
+
152
+ ## Owner guide
153
+
154
+ What the owner does once per app (or once per bucket shared by several apps on one server, each with its own prefix).
155
+
156
+ 1. **Bucket.** Create a private bucket in a region other than the server's, e.g. `acme-backups`. No public access, no
157
+ website hosting. Note its S3 endpoint (R2: `https://<account id>.r2.cloudflarestorage.com`, region `auto`; B2:
158
+ `https://s3.<region>.backblazeb2.com`; AWS: `https://s3.<region>.amazonaws.com`; Hetzner:
159
+ `https://<location>.your-objectstorage.com`).
160
+ 2. **Write-only token** for the server, limited to that bucket:
161
+ - AWS: an IAM user with only `s3:PutObject` on `arn:aws:s3:::acme-backups/<prefix>/*`.
162
+ - Backblaze B2: an application key restricted to the bucket with the single capability `writeFiles`.
163
+ - MinIO / SeaweedFS: a user or identity with a `PutObject`-only (`Write:<bucket>`) policy.
164
+ - A provider whose keys cannot be write-only (Cloudflare R2's object tokens read and write; Hetzner's keys are
165
+ per project): scope the key to the bucket and turn on the bucket's object lock / bucket lock rule for at least
166
+ the retention days, so a stolen key can neither delete nor overwrite a copy.
167
+ 3. **Read-only token** for restore and verify, separate from the write one: AWS `s3:GetObject` on the prefix plus
168
+ `s3:ListBucket` on the bucket; B2 `readFiles` + `listFiles`; R2 "Object Read only". It never goes to the server.
169
+ 4. **Lifecycle rule** (the only remote retention): `npx @softure-ai/backup lifecycle --app=<prefix> --days=30` prints
170
+ the S3 XML (`--format=json` for `aws s3api put-bucket-lifecycle-configuration`); on R2 and B2 set the same in the
171
+ bucket's settings (expire objects with prefix `<prefix>/` after 30 days).
172
+ 5. **age key pair**, on the owner's machine: `age-keygen -o backup-key.txt`. The `# public key: age1…` line is
173
+ `BACKUP_AGE_RECIPIENT`. Keep `backup-key.txt` offline (a password manager and a printed copy), never on the server
174
+ and never in the deploy's environment. Losing it makes every copy unreadable.
175
+ 6. **Secrets in the GitHub Environment** of the deploy (e.g. `production`): secrets `BACKUP_S3_ACCESS_KEY_ID`,
176
+ `BACKUP_S3_SECRET_ACCESS_KEY`; variables `BACKUP_S3_ENDPOINT`, `BACKUP_S3_REGION`, `BACKUP_S3_BUCKET`,
177
+ `BACKUP_S3_PREFIX`, `BACKUP_AGE_RECIPIENT`. The deploy job must pass them on: name the two secrets in the deploy
178
+ action's `secret-names` (or the caller's `app-secrets`) and the variables in `app-vars`
179
+ ([deploy workflow](../../tools/deploy/README.md#deploy-workflow)). The next release writes them into `.env.prod`;
180
+ the next night's `maintain` uploads.
181
+ 7. **Weekly verify:** copy [examples/backup-verify.yml](examples/backup-verify.yml) into the app's repository with its
182
+ own GitHub Environment `backup-verify` (read token, prefix, and the private key as `BACKUP_AGE_IDENTITY` if the
183
+ owner accepts it there), or run `verify` by hand on the owner's machine each week.
184
+ 8. **Alert** on the metric: `softure_backup_last_success_age_seconds > 93600` (26 h) for 15 minutes, and on
185
+ `softure_backup_last_run_ok == 0`.
186
+
187
+ ### Restore and verify
188
+
189
+ ```bash
190
+ export BACKUP_S3_ENDPOINT=… BACKUP_S3_REGION=… BACKUP_S3_BUCKET=… BACKUP_S3_PREFIX=acme-app
191
+ export BACKUP_S3_READ_ACCESS_KEY_ID=… BACKUP_S3_READ_SECRET_ACCESS_KEY=…
192
+ export BACKUP_RESTORE_DATABASE_URL=postgres://postgres:…@127.0.0.1:5432/acme_app
193
+
194
+ # The newest copy into a new database acme_app_restore_<yyyymmddhhmmss>, named after the URL's database (nothing
195
+ # existing is touched):
196
+ npx @softure-ai/backup restore --identity=backup-key.txt
197
+ # A given copy: --key=acme-app/db-20261109T031700Z.dump.age; over an existing database: --into=acme_app --overwrite
198
+
199
+ # The weekly check: restore, schema guard, row counts, drop (BACKUP_VERIFY_DATABASE_URL instead):
200
+ npx @softure-ai/backup verify --identity=- --config=deploy.json --migrations-dir=softure-migrations < backup-key.txt
201
+ ```
202
+
203
+ ### After losing the server
204
+
205
+ 1. Provision the new server as the first time (host playbook, DNS), and run the deploy workflow for the last
206
+ release's tag. That first release starts Postgres with the app's roles (`initdb/01-roles.sql`), migrates an empty
207
+ database and starts the app.
208
+ 2. On the owner's machine, decrypt the newest copies (read token and key as above):
209
+ `npx @softure-ai/backup fetch --identity=backup-key.txt --kind=database --out=db.dump`, and with a data volume
210
+ `npx @softure-ai/backup fetch --identity=backup-key.txt --kind=data --out=data.tar.gz`.
211
+ 3. Copy them to the server (`scp db.dump data.tar.gz admin@host:/tmp/`) and, as the admin user in `/srv/<app>`:
212
+ stop the app and worker (`docker compose --env-file .env.prod stop app worker`), restore over the empty database
213
+ (`docker compose --env-file .env.prod exec -T postgres pg_restore --clean --if-exists --exit-on-error -U postgres
214
+ -d <database> < /tmp/db.dump`), and unpack the data archive into the data volume
215
+ (`docker compose --env-file .env.prod run --rm --no-deps -T -v /tmp:/in --entrypoint tar app -xzf /in/data.tar.gz
216
+ -C <data path>`).
217
+ 4. Start everything again (`docker compose --env-file .env.prod up -d`), check `GET /api/health`, then delete
218
+ `/tmp/db.dump` and `/tmp/data.tar.gz` (they are not encrypted).
219
+ 5. The next night's `maintain` uploads again under the same prefix; `getLastBackup()` shows it the morning after.
220
+
221
+ ## Build
222
+
223
+ `npm run build -w modules/backup` runs `tsc -p tsconfig.build.json`. Releasing:
224
+ [scripts/release/README.md](../../scripts/release/README.md).
@@ -0,0 +1,3 @@
1
+ #!/usr/bin/env node
2
+ export {};
3
+ //# sourceMappingURL=main.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"main.d.ts","sourceRoot":"","sources":["../../src/cli/main.ts"],"names":[],"mappings":""}
@@ -0,0 +1,16 @@
1
+ #!/usr/bin/env node
2
+ import { runCli } from "./run.js";
3
+ async function readStdin() {
4
+ const chunks = [];
5
+ for await (const chunk of process.stdin)
6
+ chunks.push(typeof chunk === "string" ? Buffer.from(chunk) : chunk);
7
+ return Buffer.concat(chunks).toString("utf8");
8
+ }
9
+ process.exitCode = await runCli(process.argv.slice(2), {
10
+ cwd: process.cwd(),
11
+ env: process.env,
12
+ stdout: (text) => process.stdout.write(text),
13
+ stderr: (text) => process.stderr.write(text),
14
+ readStdin,
15
+ });
16
+ //# sourceMappingURL=main.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"main.js","sourceRoot":"","sources":["../../src/cli/main.ts"],"names":[],"mappings":";AACA,OAAO,EAAE,MAAM,EAAE,MAAM,UAAU,CAAC;AAElC,KAAK,UAAU,SAAS;IACtB,MAAM,MAAM,GAAa,EAAE,CAAC;IAC5B,IAAI,KAAK,EAAE,MAAM,KAAK,IAAI,OAAO,CAAC,KAAK;QAAE,MAAM,CAAC,IAAI,CAAC,OAAO,KAAK,KAAK,QAAQ,CAAC,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAE,KAAgB,CAAC,CAAC;IACzH,OAAO,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;AAChD,CAAC;AAED,OAAO,CAAC,QAAQ,GAAG,MAAM,MAAM,CAAC,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE;IACrD,GAAG,EAAE,OAAO,CAAC,GAAG,EAAE;IAClB,GAAG,EAAE,OAAO,CAAC,GAAG;IAChB,MAAM,EAAE,CAAC,IAAI,EAAE,EAAE,CAAC,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC;IAC5C,MAAM,EAAE,CAAC,IAAI,EAAE,EAAE,CAAC,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC;IAC5C,SAAS;CACV,CAAC,CAAC"}
@@ -0,0 +1,12 @@
1
+ export interface CliIo {
2
+ readonly cwd: string;
3
+ readonly env: Readonly<Record<string, string | undefined>>;
4
+ readonly stdout: (text: string) => void;
5
+ readonly stderr: (text: string) => void;
6
+ /** The text of stdin, read only for `--identity=-`. */
7
+ readonly readStdin: () => Promise<string>;
8
+ readonly now?: () => Date;
9
+ readonly fetch?: typeof fetch;
10
+ }
11
+ export declare function runCli(argv: readonly string[], io: CliIo): Promise<number>;
12
+ //# sourceMappingURL=run.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"run.d.ts","sourceRoot":"","sources":["../../src/cli/run.ts"],"names":[],"mappings":"AAuBA,MAAM,WAAW,KAAK;IACpB,QAAQ,CAAC,GAAG,EAAE,MAAM,CAAC;IACrB,QAAQ,CAAC,GAAG,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,GAAG,SAAS,CAAC,CAAC,CAAC;IAC3D,QAAQ,CAAC,MAAM,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,IAAI,CAAC;IACxC,QAAQ,CAAC,MAAM,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,IAAI,CAAC;IACxC,uDAAuD;IACvD,QAAQ,CAAC,SAAS,EAAE,MAAM,OAAO,CAAC,MAAM,CAAC,CAAC;IAC1C,QAAQ,CAAC,GAAG,CAAC,EAAE,MAAM,IAAI,CAAC;IAC1B,QAAQ,CAAC,KAAK,CAAC,EAAE,OAAO,KAAK,CAAC;CAC/B;AAgRD,wBAAsB,MAAM,CAAC,IAAI,EAAE,SAAS,MAAM,EAAE,EAAE,EAAE,EAAE,KAAK,GAAG,OAAO,CAAC,MAAM,CAAC,CAiChF"}
@@ -0,0 +1,335 @@
1
+ // `softure-backup <command>`: the host's nightly upload, and restore, fetch, verify and lifecycle for the operator.
2
+ // Secrets come only from the environment and the private key only from a file or stdin; an argument that looks like
3
+ // a key is refused, so nothing secret ever shows in `ps` or a shell history. Exit status: 0 done, 1 failed, 2 usage.
4
+ import { readFileSync, writeFileSync } from "node:fs";
5
+ import { resolve } from "node:path";
6
+ import { parseArgs } from "node:util";
7
+ import { parseDeployConfig, parseTableList } from "@softure-ai/deploy";
8
+ import { createLogger } from "@softure-ai/observability";
9
+ import { collectSecrets, readReadConfig, readUploadConfig, redactSecrets, } from "../offsite/config.js";
10
+ import { parseIdentities } from "../offsite/crypto.js";
11
+ import { buildLifecycleJson, buildLifecycleXml, parseLifecycleDays } from "../offsite/lifecycle.js";
12
+ import { fetchBackup, restoreBackup } from "../offsite/restore.js";
13
+ import { buildStatusSql } from "../offsite/status-sql.js";
14
+ import { buildNotConfiguredRecord, uploadBackups } from "../offsite/upload.js";
15
+ import { DEFAULT_MAX_AGE_HOURS, selectVerifyTables, verifyBackup } from "../offsite/verify.js";
16
+ const USAGE = `usage: softure-backup <command> [options]
17
+
18
+ upload --app=<name> --file=<path>... [--status-sql=<file>] [--summary=<file>]
19
+ encrypt and upload the night's backup files (BACKUP_S3_*, BACKUP_AGE_RECIPIENT)
20
+ restore --identity=<file|-> [--key=<object>] [--into=<database>] [--overwrite] [--no-owner] [--app=<name>]
21
+ restore a database backup (newest by default) into a new database (BACKUP_RESTORE_DATABASE_URL)
22
+ fetch --identity=<file|-> --out=<file> [--kind=data|database] [--key=<object>] [--app=<name>]
23
+ download and decrypt one object, e.g. the newest data archive
24
+ verify --identity=<file|-> [--config=deploy.json] [--tables=a,b] [--migrations-dir=<dir>] [--max-age-hours=26]
25
+ restore the newest copy into a throwaway database, check it, drop it (BACKUP_VERIFY_DATABASE_URL)
26
+ lifecycle [--days=30] [--format=xml|json] [--app=<name>]
27
+ print the bucket lifecycle rule that expires the prefix's objects
28
+ `;
29
+ const USAGE_EXIT = 2;
30
+ const SECRET_KEY_TEXT = /AGE-SECRET-KEY-/i;
31
+ class UsageError extends Error {
32
+ }
33
+ function readConfigOrFail(result, command) {
34
+ if (result.status === "ok")
35
+ return result.config;
36
+ if (result.status === "not_configured")
37
+ throw new UsageError(`${command}: the off-site store is not configured (BACKUP_S3_*).`);
38
+ throw new UsageError(`${command}: the off-site settings are not valid: ${result.problems.join("; ")}.`);
39
+ }
40
+ async function readIdentities(value, io, command) {
41
+ if (value === undefined)
42
+ throw new UsageError(`${command}: --identity=<file> or --identity=- (stdin) is required.`);
43
+ const text = value === "-" ? await io.readStdin() : readTextFile(resolve(io.cwd, value), command);
44
+ const parsed = parseIdentities(text);
45
+ if (!parsed.ok)
46
+ throw new UsageError(`${command}: ${parsed.problem}.`);
47
+ return parsed.identities;
48
+ }
49
+ function readTextFile(path, command) {
50
+ try {
51
+ return readFileSync(path, "utf8");
52
+ }
53
+ catch (error) {
54
+ throw new UsageError(`${command}: cannot read ${path} (${error.code ?? "unknown error"}).`);
55
+ }
56
+ }
57
+ function requireUrl(io, name, command) {
58
+ const url = io.env[name]?.trim();
59
+ if (url === undefined || url === "")
60
+ throw new UsageError(`${command}: ${name} is not set (a superuser's postgres:// URL).`);
61
+ if (!/^postgres(ql)?:\/\//.test(url))
62
+ throw new UsageError(`${command}: ${name} is not a postgres:// URL.`);
63
+ return url;
64
+ }
65
+ function getAppName(app, io, command) {
66
+ if (app !== undefined)
67
+ return app;
68
+ const prefix = io.env.BACKUP_S3_PREFIX?.trim();
69
+ if (prefix !== undefined && prefix !== "")
70
+ return prefix;
71
+ throw new UsageError(`${command}: --app=<name> is required when BACKUP_S3_PREFIX is not set.`);
72
+ }
73
+ function describeRecord(record) {
74
+ if (record.result === "ok") {
75
+ const seconds = ((record.finishedAt.getTime() - record.startedAt.getTime()) / 1000).toFixed(1);
76
+ return `ok|${String(record.objects.length)} object(s), ${String(record.sizeBytes)} bytes in ${seconds} s`;
77
+ }
78
+ if (record.result === "not_configured")
79
+ return "not_configured";
80
+ return `failed|${record.error?.code ?? "upload_failed"}|${(record.error?.message ?? "").replace(/[\r\n|]+/g, " ")}`;
81
+ }
82
+ async function runUpload(args, io) {
83
+ const { values } = parseArgs({
84
+ args,
85
+ strict: true,
86
+ options: {
87
+ app: { type: "string" },
88
+ file: { type: "string", multiple: true, default: [] },
89
+ "status-sql": { type: "string" },
90
+ summary: { type: "string" },
91
+ },
92
+ });
93
+ if (values.app === undefined)
94
+ throw new UsageError("upload: --app=<name> is required (the default key prefix).");
95
+ const now = io.now ?? (() => new Date());
96
+ const logger = createLogger("@softure-ai/backup");
97
+ const secrets = collectSecrets(io.env);
98
+ const config = readUploadConfig(io.env, values.app);
99
+ let record;
100
+ if (config.status === "not_configured") {
101
+ record = buildNotConfiguredRecord(now());
102
+ }
103
+ else if (config.status === "invalid") {
104
+ const at = now();
105
+ record = { startedAt: at, finishedAt: at, result: "failed", objects: [], sizeBytes: 0, error: { code: "config_invalid", message: config.problems.join("; ") } };
106
+ }
107
+ else {
108
+ record = await uploadBackups({
109
+ config: config.config,
110
+ files: values.file.map((file) => resolve(io.cwd, file)),
111
+ now,
112
+ s3: { now, ...(io.fetch === undefined ? {} : { fetch: io.fetch }) },
113
+ onUploaded: (object) => {
114
+ logger.info("off-site backup object uploaded", { key: object.key, bytes: object.bytes, sha256: object.sha256 });
115
+ },
116
+ });
117
+ }
118
+ if (record.error !== null)
119
+ record = { ...record, error: { ...record.error, message: redactSecrets(record.error.message, secrets) } };
120
+ if (values["status-sql"] !== undefined)
121
+ writeFileSync(resolve(io.cwd, values["status-sql"]), buildStatusSql(record), { mode: 0o600 });
122
+ const summary = describeRecord(record);
123
+ if (values.summary !== undefined)
124
+ writeFileSync(resolve(io.cwd, values.summary), `${summary}\n`, { mode: 0o600 });
125
+ const durationMs = record.finishedAt.getTime() - record.startedAt.getTime();
126
+ if (record.result === "failed") {
127
+ logger.error("off-site backup failed", { result: record.result, code: record.error?.code, message: record.error?.message, durationMs });
128
+ io.stderr(`upload: failed (${record.error?.code ?? ""}): ${record.error?.message ?? ""}\n`);
129
+ return 1;
130
+ }
131
+ if (record.result === "not_configured") {
132
+ io.stdout("upload: not configured (no BACKUP_S3_* in the environment); nothing uploaded.\n");
133
+ return 0;
134
+ }
135
+ logger.info("off-site backup succeeded", { result: record.result, objects: record.objects.length, bytes: record.sizeBytes, durationMs });
136
+ for (const object of record.objects)
137
+ io.stdout(`upload: ${object.key} (${String(object.bytes)} bytes, sha256 ${object.sha256})\n`);
138
+ return 0;
139
+ }
140
+ async function runRestore(args, io) {
141
+ const { values } = parseArgs({
142
+ args,
143
+ strict: true,
144
+ options: {
145
+ identity: { type: "string" },
146
+ key: { type: "string" },
147
+ into: { type: "string" },
148
+ overwrite: { type: "boolean", default: false },
149
+ "no-owner": { type: "boolean", default: false },
150
+ app: { type: "string" },
151
+ "pg-restore": { type: "string" },
152
+ },
153
+ });
154
+ const store = readConfigOrFail(readReadConfig(io.env, getAppName(values.app, io, "restore")), "restore");
155
+ const adminUrl = requireUrl(io, "BACKUP_RESTORE_DATABASE_URL", "restore");
156
+ if (values.overwrite && values.into === undefined)
157
+ throw new UsageError("restore: --overwrite needs --into=<database>, the database to restore over.");
158
+ const identities = await readIdentities(values.identity, io, "restore");
159
+ const result = await restoreBackup({
160
+ store,
161
+ identities,
162
+ adminUrl,
163
+ ...(values.key === undefined ? {} : { key: values.key }),
164
+ ...(values.into === undefined ? {} : { into: values.into }),
165
+ overwrite: values.overwrite,
166
+ noOwner: values["no-owner"],
167
+ pgRestore: values["pg-restore"] ?? io.env.BACKUP_PG_RESTORE ?? "pg_restore",
168
+ env: io.env,
169
+ ...(io.now === undefined ? {} : { now: io.now() }),
170
+ s3: io.fetch === undefined ? {} : { fetch: io.fetch },
171
+ });
172
+ if (!result.ok) {
173
+ io.stderr(`restore: ${redactSecrets(result.problem, collectSecrets(io.env))}\n`);
174
+ return 1;
175
+ }
176
+ io.stdout(`restore: ${result.value.key} restored into ${result.value.created ? "the new database" : "the existing database"} ${result.value.database}\n`);
177
+ return 0;
178
+ }
179
+ async function runFetch(args, io) {
180
+ const { values } = parseArgs({
181
+ args,
182
+ strict: true,
183
+ options: {
184
+ identity: { type: "string" },
185
+ key: { type: "string" },
186
+ kind: { type: "string", default: "data" },
187
+ out: { type: "string" },
188
+ app: { type: "string" },
189
+ },
190
+ });
191
+ if (values.kind !== "data" && values.kind !== "database")
192
+ throw new UsageError("fetch: --kind is data or database.");
193
+ if (values.out === undefined)
194
+ throw new UsageError("fetch: --out=<file> is required.");
195
+ const store = readConfigOrFail(readReadConfig(io.env, getAppName(values.app, io, "fetch")), "fetch");
196
+ const identities = await readIdentities(values.identity, io, "fetch");
197
+ const result = await fetchBackup({
198
+ store,
199
+ identities,
200
+ kind: values.kind,
201
+ out: resolve(io.cwd, values.out),
202
+ ...(values.key === undefined ? {} : { key: values.key }),
203
+ s3: io.fetch === undefined ? {} : { fetch: io.fetch },
204
+ });
205
+ if (!result.ok) {
206
+ io.stderr(`fetch: ${redactSecrets(result.problem, collectSecrets(io.env))}\n`);
207
+ return 1;
208
+ }
209
+ io.stdout(`fetch: ${result.value.key} decrypted to ${result.value.out}\n`);
210
+ return 0;
211
+ }
212
+ function readVerifyTables(config, tables, io) {
213
+ if (tables !== undefined) {
214
+ const parsed = parseTableList(tables);
215
+ if (!parsed.ok)
216
+ throw new UsageError(`verify: --tables: ${parsed.problem}.`);
217
+ return parsed.tables;
218
+ }
219
+ if (config === undefined)
220
+ return [];
221
+ const path = resolve(io.cwd, config);
222
+ let json;
223
+ try {
224
+ json = JSON.parse(readTextFile(path, "verify"));
225
+ }
226
+ catch (error) {
227
+ if (error instanceof UsageError)
228
+ throw error;
229
+ throw new UsageError(`verify: ${path} is not JSON.`);
230
+ }
231
+ const parsed = parseDeployConfig(json);
232
+ if (!parsed.ok)
233
+ throw new UsageError(`verify: ${path} is not a valid deploy.json: ${parsed.issues.join("; ")}.`);
234
+ return selectVerifyTables(parsed.config.database?.rowCountTables, parsed.config.database?.excludeTableData);
235
+ }
236
+ async function runVerify(args, io) {
237
+ const { values } = parseArgs({
238
+ args,
239
+ strict: true,
240
+ options: {
241
+ identity: { type: "string" },
242
+ config: { type: "string" },
243
+ tables: { type: "string" },
244
+ "migrations-dir": { type: "string" },
245
+ "max-age-hours": { type: "string" },
246
+ app: { type: "string" },
247
+ "pg-restore": { type: "string" },
248
+ },
249
+ });
250
+ const maxAgeText = values["max-age-hours"];
251
+ const maxAgeHours = maxAgeText === undefined ? DEFAULT_MAX_AGE_HOURS : Number(maxAgeText);
252
+ if (!Number.isFinite(maxAgeHours) || maxAgeHours <= 0 || maxAgeHours > 24 * 366)
253
+ throw new UsageError("verify: --max-age-hours is a number of hours above 0.");
254
+ const store = readConfigOrFail(readReadConfig(io.env, getAppName(values.app, io, "verify")), "verify");
255
+ const adminUrl = requireUrl(io, "BACKUP_VERIFY_DATABASE_URL", "verify");
256
+ const tables = readVerifyTables(values.config, values.tables, io);
257
+ const identities = await readIdentities(values.identity, io, "verify");
258
+ const logger = createLogger("@softure-ai/backup");
259
+ const result = await verifyBackup({
260
+ store,
261
+ identities,
262
+ adminUrl,
263
+ tables,
264
+ ...(values["migrations-dir"] === undefined ? {} : { migrationsDir: resolve(io.cwd, values["migrations-dir"]) }),
265
+ maxAgeHours,
266
+ pgRestore: values["pg-restore"] ?? io.env.BACKUP_PG_RESTORE ?? "pg_restore",
267
+ env: io.env,
268
+ ...(io.now === undefined ? {} : { now: io.now() }),
269
+ s3: io.fetch === undefined ? {} : { fetch: io.fetch },
270
+ });
271
+ if (!result.ok) {
272
+ const problem = redactSecrets(result.problem, collectSecrets(io.env));
273
+ logger.error("off-site backup verify failed", { problem });
274
+ io.stderr(`verify: ${problem}\n`);
275
+ return 1;
276
+ }
277
+ const counts = Object.entries(result.value.counts).map(([table, count]) => `${table}=${String(count)}`).join(",");
278
+ logger.info("off-site backup verified", { key: result.value.key, ageHours: Number(result.value.ageHours.toFixed(1)), schemaGuard: result.value.schemaGuard });
279
+ io.stdout(`verify: ${result.value.key} (${result.value.ageHours.toFixed(1)} h old) restores; schema guard ${result.value.schemaGuard}; ${counts === "" ? "no tables listed" : counts}\n`);
280
+ return 0;
281
+ }
282
+ function runLifecycle(args, io) {
283
+ const { values } = parseArgs({
284
+ args,
285
+ strict: true,
286
+ options: { days: { type: "string" }, format: { type: "string", default: "xml" }, app: { type: "string" } },
287
+ });
288
+ const days = parseLifecycleDays(values.days);
289
+ if (!days.ok)
290
+ throw new UsageError(`lifecycle: ${days.problem}.`);
291
+ const prefix = getAppName(values.app, io, "lifecycle");
292
+ if (values.format === "xml")
293
+ io.stdout(buildLifecycleXml(prefix, days.days));
294
+ else if (values.format === "json")
295
+ io.stdout(buildLifecycleJson(prefix, days.days));
296
+ else
297
+ throw new UsageError("lifecycle: --format is xml or json.");
298
+ return 0;
299
+ }
300
+ export async function runCli(argv, io) {
301
+ const [command, ...args] = argv;
302
+ try {
303
+ if (args.some((arg) => SECRET_KEY_TEXT.test(arg))) {
304
+ throw new UsageError("an argument holds an age secret key; pass it through --identity=<file> or --identity=- (stdin), never as an argument.");
305
+ }
306
+ switch (command) {
307
+ case "upload":
308
+ return await runUpload(args, io);
309
+ case "restore":
310
+ return await runRestore(args, io);
311
+ case "fetch":
312
+ return await runFetch(args, io);
313
+ case "verify":
314
+ return await runVerify(args, io);
315
+ case "lifecycle":
316
+ return runLifecycle(args, io);
317
+ case undefined:
318
+ case "--help":
319
+ case "help":
320
+ io.stdout(USAGE);
321
+ return command === undefined ? USAGE_EXIT : 0;
322
+ default:
323
+ throw new UsageError(`unknown command ${command}.\n${USAGE}`);
324
+ }
325
+ }
326
+ catch (error) {
327
+ if (error instanceof UsageError || (error instanceof TypeError && "code" in error && String(error.code).startsWith("ERR_PARSE_ARGS"))) {
328
+ io.stderr(`softure-backup: ${error.message}\n`);
329
+ return USAGE_EXIT;
330
+ }
331
+ io.stderr(`softure-backup: ${command ?? ""} failed: ${redactSecrets(error instanceof Error ? error.message : String(error), collectSecrets(io.env))}\n`);
332
+ return 1;
333
+ }
334
+ }
335
+ //# sourceMappingURL=run.js.map