@caelo-cms/provisioning 0.10.48 → 0.10.50
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/caddy/Caddyfile.production +5 -1
- package/caddy/Caddyfile.staging +5 -1
- package/dist/agent-db.js +102 -58
- package/dist/caddy-redirects.d.ts +15 -0
- package/dist/caddy-redirects.d.ts.map +1 -0
- package/dist/caddy-redirects.js +65 -0
- package/dist/caddy-redirects.js.map +1 -0
- package/dist/caddy.d.ts +22 -2
- package/dist/caddy.d.ts.map +1 -1
- package/dist/caddy.js +66 -17
- package/dist/caddy.js.map +1 -1
- package/dist/cli-routing.d.ts.map +1 -1
- package/dist/cli-routing.js +7 -0
- package/dist/cli-routing.js.map +1 -1
- package/dist/cli.js +248 -254
- package/dist/cli.js.map +1 -1
- package/dist/compose.d.ts +54 -2
- package/dist/compose.d.ts.map +1 -1
- package/dist/compose.js +86 -32
- package/dist/compose.js.map +1 -1
- package/dist/gcp-names.d.ts +12 -0
- package/dist/gcp-names.d.ts.map +1 -1
- package/dist/gcp-names.js +21 -0
- package/dist/gcp-names.js.map +1 -1
- package/dist/gcp-url-map.d.ts +27 -0
- package/dist/gcp-url-map.d.ts.map +1 -0
- package/dist/gcp-url-map.js +62 -0
- package/dist/gcp-url-map.js.map +1 -0
- package/dist/index.d.ts +0 -1
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +0 -1
- package/dist/index.js.map +1 -1
- package/dist/infra-contract.d.ts +20 -1
- package/dist/infra-contract.d.ts.map +1 -1
- package/dist/infra-contract.js +39 -1
- package/dist/infra-contract.js.map +1 -1
- package/dist/public-role-isolation.d.ts +19 -0
- package/dist/public-role-isolation.d.ts.map +1 -0
- package/dist/public-role-isolation.js +69 -0
- package/dist/public-role-isolation.js.map +1 -0
- package/dist/release-contract.d.ts.map +1 -1
- package/dist/release-contract.js +3 -0
- package/dist/release-contract.js.map +1 -1
- package/dist/release-manifest.d.ts +2 -41
- package/dist/release-manifest.d.ts.map +1 -1
- package/dist/release-manifest.js +20 -122
- package/dist/release-manifest.js.map +1 -1
- package/dist/self-hosted-backup.d.ts +86 -0
- package/dist/self-hosted-backup.d.ts.map +1 -0
- package/dist/self-hosted-backup.js +350 -0
- package/dist/self-hosted-backup.js.map +1 -0
- package/dist/self-hosted-release.d.ts +12 -3
- package/dist/self-hosted-release.d.ts.map +1 -1
- package/dist/self-hosted-release.js +6 -4
- package/dist/self-hosted-release.js.map +1 -1
- package/dist/self-hosted-start.d.ts +50 -0
- package/dist/self-hosted-start.d.ts.map +1 -0
- package/dist/self-hosted-start.js +59 -0
- package/dist/self-hosted-start.js.map +1 -0
- package/dist/self-hosted-upgrade.d.ts +30 -0
- package/dist/self-hosted-upgrade.d.ts.map +1 -1
- package/dist/self-hosted-upgrade.js +29 -7
- package/dist/self-hosted-upgrade.js.map +1 -1
- package/dist/tar-entries.d.ts +14 -0
- package/dist/tar-entries.d.ts.map +1 -0
- package/dist/tar-entries.js +104 -0
- package/dist/tar-entries.js.map +1 -0
- package/dist/upgrade/converge.d.ts +2 -2
- package/dist/upgrade/converge.d.ts.map +1 -1
- package/dist/upgrade/converge.js +44 -2
- package/dist/upgrade/converge.js.map +1 -1
- package/dist/upgrade/release-images.d.ts +14 -2
- package/dist/upgrade/release-images.d.ts.map +1 -1
- package/dist/upgrade/release-images.js +4 -2
- package/dist/upgrade/release-images.js.map +1 -1
- package/dist/upgrader-assess.d.ts +4 -1
- package/dist/upgrader-assess.d.ts.map +1 -1
- package/dist/upgrader-assess.js +8 -1
- package/dist/upgrader-assess.js.map +1 -1
- package/dist/upgrader-job.d.ts +3 -1
- package/dist/upgrader-job.d.ts.map +1 -1
- package/dist/upgrader-job.js.map +1 -1
- package/dist/url-map-redirects-access.d.ts +15 -0
- package/dist/url-map-redirects-access.d.ts.map +1 -0
- package/dist/url-map-redirects-access.js +81 -0
- package/dist/url-map-redirects-access.js.map +1 -0
- package/dist/wizard.d.ts.map +1 -1
- package/dist/wizard.js +3 -1
- package/dist/wizard.js.map +1 -1
- package/dist/wizards/gcp.d.ts.map +1 -1
- package/dist/wizards/gcp.js +45 -0
- package/dist/wizards/gcp.js.map +1 -1
- package/package.json +2 -2
- package/src/admin-memory.test.ts +2 -2
- package/src/agent/run.test.ts +55 -0
- package/src/caddy-redirects.docker.test.ts +255 -0
- package/src/caddy-redirects.test.ts +115 -0
- package/src/caddy-redirects.ts +68 -0
- package/src/caddy-validate.test.ts +119 -0
- package/src/caddy.test.ts +108 -0
- package/src/caddy.ts +89 -20
- package/src/cli-routing.test.ts +12 -0
- package/src/cli-routing.ts +7 -0
- package/src/cli.ts +299 -287
- package/src/compose.ts +106 -34
- package/src/gcp-names.ts +23 -0
- package/src/gcp-url-map.test.ts +150 -0
- package/src/gcp-url-map.ts +81 -0
- package/src/index.test.ts +18 -146
- package/src/index.ts +0 -9
- package/src/infra-contract.test.ts +39 -0
- package/src/infra-contract.ts +42 -1
- package/src/public-role-isolation.test.ts +89 -0
- package/src/public-role-isolation.ts +83 -0
- package/src/release-contract.ts +3 -0
- package/src/release-manifest.test.ts +20 -1
- package/src/release-manifest.ts +39 -158
- package/src/self-hosted-backup.test.ts +344 -0
- package/src/self-hosted-backup.ts +480 -0
- package/src/self-hosted-compose.test.ts +88 -4
- package/src/self-hosted-release.ts +22 -7
- package/src/self-hosted-start.test.ts +68 -0
- package/src/self-hosted-start.ts +123 -0
- package/src/self-hosted-upgrade.ts +35 -8
- package/src/stack-contract.test.ts +2 -2
- package/src/tar-entries.test.ts +52 -0
- package/src/tar-entries.ts +116 -0
- package/src/upgrade/converge.ts +51 -4
- package/src/upgrade/release-images.ts +4 -2
- package/src/upgrade/upgrade.test.ts +78 -1
- package/src/upgrader-assess.test.ts +64 -1
- package/src/upgrader-assess.ts +20 -2
- package/src/upgrader-job.test.ts +3 -0
- package/src/upgrader-job.ts +3 -0
- package/src/url-map-redirects-access.test.ts +88 -0
- package/src/url-map-redirects-access.ts +89 -0
- package/src/wizard.ts +3 -1
- package/src/wizards/gcp.ts +48 -0
- package/stacks/aws/index.ts +34 -2
- package/stacks/azure/index.ts +13 -0
- package/stacks/gcp/index.ts +24 -17
- package/stacks/gcp-firebase/index.ts +7 -0
- package/stacks/self-hosted/Pulumi.yaml +2 -2
- package/stacks/self-hosted/README.md +3 -3
- package/stacks/self-hosted/index.ts +45 -22
- package/dist/redirects-emit.d.ts +0 -52
- package/dist/redirects-emit.d.ts.map +0 -1
- package/dist/redirects-emit.js +0 -111
- package/dist/redirects-emit.js.map +0 -1
- package/src/redirects-emit.ts +0 -175
|
@@ -0,0 +1,480 @@
|
|
|
1
|
+
// SPDX-License-Identifier: MPL-2.0
|
|
2
|
+
|
|
3
|
+
/**
|
|
4
|
+
* `cms-provision backup` / `restore` for a self-hosted Compose install
|
|
5
|
+
* (issue #707).
|
|
6
|
+
*
|
|
7
|
+
* A backup is one `.tar.gz` holding everything an install's content lives
|
|
8
|
+
* in:
|
|
9
|
+
* manifest.json format version, Caelo version, timestamp
|
|
10
|
+
* db/cms_admin.dump pg_dump -Fc (the same dumps `upgrade` takes)
|
|
11
|
+
* db/cms_public.dump
|
|
12
|
+
* db/globals.sql the cluster's roles — reference only, see restore
|
|
13
|
+
* files.tar the published site (`/srv/caelo`) and the uploaded
|
|
14
|
+
* media (`/data/media`), read through the admin's
|
|
15
|
+
* own mounts
|
|
16
|
+
* Secrets (`.caelo/config.json`, incl. the KEK that decrypts stored
|
|
17
|
+
* provider keys) are NOT in the archive: it may be copied off-host, and a
|
|
18
|
+
* restore targets the install whose config is already there.
|
|
19
|
+
*
|
|
20
|
+
* pg_dump rather than pgBackRest: the old sidecar never had access to the
|
|
21
|
+
* data volume and no stanza was ever created, so it could not back up
|
|
22
|
+
* anything. WAL streaming / point-in-time recovery (CMS_REQUIREMENTS §13.2)
|
|
23
|
+
* needs pgBackRest inside the Postgres image and is tracked separately.
|
|
24
|
+
*
|
|
25
|
+
* Files go through a one-off admin container (`docker compose run`), not
|
|
26
|
+
* the host: the admin writes them as root, which the operator's user may
|
|
27
|
+
* not be able to replace.
|
|
28
|
+
*
|
|
29
|
+
* Backup stops the admin and the gateway for its duration (a short outage):
|
|
30
|
+
* the two databases and the files are separate steps, and a write between
|
|
31
|
+
* them would give an archive whose parts do not match each other.
|
|
32
|
+
*
|
|
33
|
+
* Restore order: list + check the archive and its files.tar WITHOUT
|
|
34
|
+
* extracting (only the expected members, relative paths, no member below a
|
|
35
|
+
* symlink — nothing on the install changes otherwise) → stop the Caelo
|
|
36
|
+
* services → drop + re-create both databases with the role convergence
|
|
37
|
+
* (roles, passwords and grants come from config.json, never from the
|
|
38
|
+
* archive, so a restore cannot reset the superuser's password) →
|
|
39
|
+
* pg_restore → converge again (re-applies the gateway grants the restored
|
|
40
|
+
* functions define) → migrate with the install's admin image (a backup of
|
|
41
|
+
* an older release has its older schema) → replace the files → start
|
|
42
|
+
* everything and health-check it → re-render + reload the Caddyfile from
|
|
43
|
+
* the restored domains table.
|
|
44
|
+
*/
|
|
45
|
+
|
|
46
|
+
import {
|
|
47
|
+
chmodSync,
|
|
48
|
+
existsSync,
|
|
49
|
+
mkdirSync,
|
|
50
|
+
readFileSync,
|
|
51
|
+
rmSync,
|
|
52
|
+
statSync,
|
|
53
|
+
writeFileSync,
|
|
54
|
+
} from "node:fs";
|
|
55
|
+
import { basename, join } from "node:path";
|
|
56
|
+
import { gunzipSync } from "node:zlib";
|
|
57
|
+
import { COMPOSE_APP_SERVICES, MEDIA_ROOT, SITE_OUTPUT_ROOT } from "./compose.js";
|
|
58
|
+
import type { HostRunner } from "./self-hosted-release.js";
|
|
59
|
+
import type { ConvergeOutcome } from "./self-hosted-roles.js";
|
|
60
|
+
import {
|
|
61
|
+
compose,
|
|
62
|
+
DATABASE_DUMPS,
|
|
63
|
+
dumpDatabases,
|
|
64
|
+
runMigrations,
|
|
65
|
+
waitHealthy,
|
|
66
|
+
} from "./self-hosted-upgrade.js";
|
|
67
|
+
import { escapesRoot, listTarEntries, normalizeMemberName, type TarEntry } from "./tar-entries.js";
|
|
68
|
+
|
|
69
|
+
/** The archive layout's version; a restore refuses any other. */
|
|
70
|
+
export const BACKUP_FORMAT = 1;
|
|
71
|
+
|
|
72
|
+
const MANIFEST = "manifest.json";
|
|
73
|
+
const DB_DIR = "db";
|
|
74
|
+
const FILES_TAR = "files.tar";
|
|
75
|
+
|
|
76
|
+
/** The content directories inside the admin container, as `tar -C /` paths. */
|
|
77
|
+
const CONTENT_PATHS = [SITE_OUTPUT_ROOT, MEDIA_ROOT].map((p) => p.replace(/^\//, ""));
|
|
78
|
+
|
|
79
|
+
export interface BackupManifest {
|
|
80
|
+
readonly format: number;
|
|
81
|
+
readonly caeloVersion: string;
|
|
82
|
+
readonly createdAt: string;
|
|
83
|
+
}
|
|
84
|
+
|
|
85
|
+
export interface BackupDeps {
|
|
86
|
+
readonly caeloDir: string;
|
|
87
|
+
readonly runner: HostRunner;
|
|
88
|
+
readonly log: (message: string) => void;
|
|
89
|
+
readonly now?: () => Date;
|
|
90
|
+
readonly healthAttempts?: number;
|
|
91
|
+
readonly healthIntervalMs?: number;
|
|
92
|
+
readonly sleep?: (ms: number) => Promise<void>;
|
|
93
|
+
}
|
|
94
|
+
|
|
95
|
+
export type BackupResult = { ok: true; path: string } | { ok: false; error: string };
|
|
96
|
+
|
|
97
|
+
/**
|
|
98
|
+
* Write a backup archive of the install to `to` (mode 600). Postgres must
|
|
99
|
+
* be running. The admin and the gateway are stopped while it runs and
|
|
100
|
+
* started (and health-checked) again afterwards, whatever the outcome.
|
|
101
|
+
*/
|
|
102
|
+
export async function backupSelfHosted(
|
|
103
|
+
to: string,
|
|
104
|
+
caeloVersion: string,
|
|
105
|
+
deps: BackupDeps,
|
|
106
|
+
): Promise<BackupResult> {
|
|
107
|
+
const { caeloDir, runner, log } = deps;
|
|
108
|
+
const at = (deps.now ?? (() => new Date()))();
|
|
109
|
+
const ready = await runner.run(
|
|
110
|
+
compose(caeloDir, "exec", "-T", "postgres", "pg_isready", "-U", "caelo"),
|
|
111
|
+
);
|
|
112
|
+
if (ready.exitCode !== 0) {
|
|
113
|
+
return {
|
|
114
|
+
ok: false,
|
|
115
|
+
error: `Postgres is not running, so nothing was backed up. Start it with \`docker compose -f ${caeloDir}/docker-compose.yml up -d postgres\` and re-run.`,
|
|
116
|
+
};
|
|
117
|
+
}
|
|
118
|
+
log("Stopping the admin and the gateway for a consistent snapshot …");
|
|
119
|
+
const stop = await runner.run(compose(caeloDir, "stop", ...COMPOSE_APP_SERVICES));
|
|
120
|
+
if (stop.exitCode !== 0) {
|
|
121
|
+
return { ok: false, error: `Could not stop the Caelo services: ${tail(stop.stderr)}` };
|
|
122
|
+
}
|
|
123
|
+
const work = workDir(caeloDir, "backup", at);
|
|
124
|
+
let result: BackupResult;
|
|
125
|
+
try {
|
|
126
|
+
result = await writeArchive(to, caeloVersion, work, deps);
|
|
127
|
+
} finally {
|
|
128
|
+
rmSync(work, { recursive: true, force: true });
|
|
129
|
+
}
|
|
130
|
+
log("Starting the admin and the gateway again …");
|
|
131
|
+
const started = await startAppServices(deps);
|
|
132
|
+
if (!started.ok) {
|
|
133
|
+
return {
|
|
134
|
+
ok: false,
|
|
135
|
+
error: `${result.ok ? `The backup was written to ${result.path}, but` : `${result.error}\nAlso,`} the services did not come back: ${started.error}`,
|
|
136
|
+
};
|
|
137
|
+
}
|
|
138
|
+
return result;
|
|
139
|
+
}
|
|
140
|
+
|
|
141
|
+
/** The dumps, files.tar and manifest into `work`, then one .tar.gz at `to`. */
|
|
142
|
+
async function writeArchive(
|
|
143
|
+
to: string,
|
|
144
|
+
caeloVersion: string,
|
|
145
|
+
work: string,
|
|
146
|
+
deps: BackupDeps,
|
|
147
|
+
): Promise<BackupResult> {
|
|
148
|
+
const { caeloDir, runner, log } = deps;
|
|
149
|
+
log("Dumping the databases (pg_dump) …");
|
|
150
|
+
const dumped = await dumpDatabases(caeloDir, runner, join(work, DB_DIR));
|
|
151
|
+
if (!dumped.ok) return { ok: false, error: dumped.error };
|
|
152
|
+
|
|
153
|
+
log("Archiving the published site and the uploaded media …");
|
|
154
|
+
const files = await runner.run(
|
|
155
|
+
compose(
|
|
156
|
+
caeloDir,
|
|
157
|
+
"run",
|
|
158
|
+
"--rm",
|
|
159
|
+
"--no-deps",
|
|
160
|
+
"-T",
|
|
161
|
+
"--entrypoint",
|
|
162
|
+
"tar",
|
|
163
|
+
"caelo-admin",
|
|
164
|
+
"-C",
|
|
165
|
+
"/",
|
|
166
|
+
"-cf",
|
|
167
|
+
"-",
|
|
168
|
+
...CONTENT_PATHS,
|
|
169
|
+
),
|
|
170
|
+
{ stdoutPath: join(work, FILES_TAR) },
|
|
171
|
+
);
|
|
172
|
+
if (files.exitCode !== 0) {
|
|
173
|
+
return { ok: false, error: `Could not archive the site + media files: ${tail(files.stderr)}` };
|
|
174
|
+
}
|
|
175
|
+
|
|
176
|
+
const manifest: BackupManifest = {
|
|
177
|
+
format: BACKUP_FORMAT,
|
|
178
|
+
caeloVersion,
|
|
179
|
+
createdAt: (deps.now ?? (() => new Date()))().toISOString(),
|
|
180
|
+
};
|
|
181
|
+
writeFileSync(join(work, MANIFEST), `${JSON.stringify(manifest, null, 2)}\n`, { mode: 0o600 });
|
|
182
|
+
|
|
183
|
+
log(`Writing ${to} …`);
|
|
184
|
+
// COPYFILE_DISABLE: macOS's bsdtar would add `._*` AppleDouble members
|
|
185
|
+
// for extended attributes, which a restore refuses as unexpected.
|
|
186
|
+
const tar = await runner.run(["tar", "-czf", to, "-C", work, "."], {
|
|
187
|
+
env: { COPYFILE_DISABLE: "1" },
|
|
188
|
+
});
|
|
189
|
+
if (tar.exitCode !== 0) {
|
|
190
|
+
return { ok: false, error: `Could not write ${to}: ${tail(tar.stderr)}` };
|
|
191
|
+
}
|
|
192
|
+
chmodSync(to, 0o600);
|
|
193
|
+
return { ok: true, path: to };
|
|
194
|
+
}
|
|
195
|
+
|
|
196
|
+
/** `up -d` the admin + gateway, then health-check both. */
|
|
197
|
+
async function startAppServices(
|
|
198
|
+
deps: BackupDeps,
|
|
199
|
+
): Promise<{ ok: true } | { ok: false; error: string }> {
|
|
200
|
+
const { caeloDir, runner } = deps;
|
|
201
|
+
const up = await runner.run(compose(caeloDir, "up", "-d", ...COMPOSE_APP_SERVICES));
|
|
202
|
+
if (up.exitCode !== 0)
|
|
203
|
+
return { ok: false, error: `docker compose up failed: ${tail(up.stderr)}` };
|
|
204
|
+
for (const service of ["admin", "gateway"] as const) {
|
|
205
|
+
const healthy = await waitHealthy(service, caeloDir, {
|
|
206
|
+
runner,
|
|
207
|
+
...(deps.sleep ? { sleep: deps.sleep } : {}),
|
|
208
|
+
...(deps.healthAttempts !== undefined ? { healthAttempts: deps.healthAttempts } : {}),
|
|
209
|
+
...(deps.healthIntervalMs !== undefined ? { healthIntervalMs: deps.healthIntervalMs } : {}),
|
|
210
|
+
});
|
|
211
|
+
if (!healthy.ok) return { ok: false, error: `the ${service} is not healthy: ${healthy.error}` };
|
|
212
|
+
}
|
|
213
|
+
return { ok: true };
|
|
214
|
+
}
|
|
215
|
+
|
|
216
|
+
export interface RestoreDeps extends BackupDeps {
|
|
217
|
+
/** Brings roles, databases and grants in line with config.json (self-hosted-roles.ts). */
|
|
218
|
+
readonly convergeRoles: () => Promise<ConvergeOutcome>;
|
|
219
|
+
/**
|
|
220
|
+
* Re-renders the Caddyfile from the (restored) domains table and reloads
|
|
221
|
+
* Caddy, so the served hosts match the restored database.
|
|
222
|
+
*/
|
|
223
|
+
readonly refreshCaddy: () => Promise<{ ok: true } | { ok: false; error: string }>;
|
|
224
|
+
}
|
|
225
|
+
|
|
226
|
+
export type RestoreResult =
|
|
227
|
+
| { ok: true; manifest: BackupManifest }
|
|
228
|
+
| { ok: false; error: string; installChanged: boolean };
|
|
229
|
+
|
|
230
|
+
/** Replace the install's databases and content files with the backup at `from`. */
|
|
231
|
+
export async function restoreSelfHosted(from: string, deps: RestoreDeps): Promise<RestoreResult> {
|
|
232
|
+
const { caeloDir, runner, log } = deps;
|
|
233
|
+
if (!existsSync(from)) return unchanged(`${from} does not exist.`);
|
|
234
|
+
const work = workDir(caeloDir, "restore", (deps.now ?? (() => new Date()))());
|
|
235
|
+
try {
|
|
236
|
+
log(`Checking ${basename(from)} …`);
|
|
237
|
+
const members = checkArchiveMembers(from);
|
|
238
|
+
if (!members.ok) return unchanged(members.error);
|
|
239
|
+
const untar = await runner.run(["tar", "-xzf", from, "-C", work]);
|
|
240
|
+
if (untar.exitCode !== 0) {
|
|
241
|
+
return unchanged(`${from} is not a readable backup archive: ${tail(untar.stderr)}`);
|
|
242
|
+
}
|
|
243
|
+
const checked = checkArchive(work);
|
|
244
|
+
if (!checked.ok) return unchanged(checked.error);
|
|
245
|
+
const files = checkFilesTar(join(work, FILES_TAR));
|
|
246
|
+
if (!files.ok) return unchanged(files.error);
|
|
247
|
+
|
|
248
|
+
log("Stopping the admin and the gateway …");
|
|
249
|
+
const stop = await runner.run(compose(caeloDir, "stop", ...COMPOSE_APP_SERVICES));
|
|
250
|
+
if (stop.exitCode !== 0) {
|
|
251
|
+
return unchanged(`Could not stop the Caelo services: ${tail(stop.stderr)}`);
|
|
252
|
+
}
|
|
253
|
+
const pg = await runner.run(compose(caeloDir, "up", "-d", "--wait", "postgres"));
|
|
254
|
+
if (pg.exitCode !== 0) {
|
|
255
|
+
return changed(`Postgres did not become healthy: ${tail(pg.stderr)}`);
|
|
256
|
+
}
|
|
257
|
+
|
|
258
|
+
log("Replacing the databases …");
|
|
259
|
+
const drop = await runner.run(
|
|
260
|
+
compose(
|
|
261
|
+
caeloDir,
|
|
262
|
+
"exec",
|
|
263
|
+
"-T",
|
|
264
|
+
"postgres",
|
|
265
|
+
"psql",
|
|
266
|
+
"-U",
|
|
267
|
+
"caelo",
|
|
268
|
+
"-d",
|
|
269
|
+
"postgres",
|
|
270
|
+
"-v",
|
|
271
|
+
"ON_ERROR_STOP=1",
|
|
272
|
+
"-c",
|
|
273
|
+
"DROP DATABASE IF EXISTS cms_admin WITH (FORCE)",
|
|
274
|
+
"-c",
|
|
275
|
+
"DROP DATABASE IF EXISTS cms_public WITH (FORCE)",
|
|
276
|
+
),
|
|
277
|
+
);
|
|
278
|
+
if (drop.exitCode !== 0) return changed(`Could not drop the databases: ${tail(drop.stderr)}`);
|
|
279
|
+
const created = await deps.convergeRoles();
|
|
280
|
+
if (!created.ok) return changed(`Could not re-create the databases: ${created.error}`);
|
|
281
|
+
for (const db of ["cms_admin", "cms_public"] as const) {
|
|
282
|
+
const r = await runner.run(
|
|
283
|
+
compose(
|
|
284
|
+
caeloDir,
|
|
285
|
+
"exec",
|
|
286
|
+
"-T",
|
|
287
|
+
"postgres",
|
|
288
|
+
"pg_restore",
|
|
289
|
+
"-U",
|
|
290
|
+
"caelo",
|
|
291
|
+
"-d",
|
|
292
|
+
db,
|
|
293
|
+
"--exit-on-error",
|
|
294
|
+
"--single-transaction",
|
|
295
|
+
),
|
|
296
|
+
{ stdinPath: join(work, DB_DIR, DATABASE_DUMPS[db]) },
|
|
297
|
+
);
|
|
298
|
+
if (r.exitCode !== 0) return changed(`pg_restore of ${db} failed: ${tail(r.stderr)}`);
|
|
299
|
+
}
|
|
300
|
+
const granted = await deps.convergeRoles();
|
|
301
|
+
if (!granted.ok) return changed(`Could not re-apply the role grants: ${granted.error}`);
|
|
302
|
+
log("Applying the migrations of the running release …");
|
|
303
|
+
const migrated = await runMigrations(caeloDir, runner);
|
|
304
|
+
if (!migrated.ok) return changed(migrated.error);
|
|
305
|
+
|
|
306
|
+
log("Replacing the published site and the uploaded media …");
|
|
307
|
+
const dirs = CONTENT_PATHS.map((p) => `/${p}`).join(" ");
|
|
308
|
+
const replaced = await runner.run(
|
|
309
|
+
compose(
|
|
310
|
+
caeloDir,
|
|
311
|
+
"run",
|
|
312
|
+
"--rm",
|
|
313
|
+
"--no-deps",
|
|
314
|
+
"-T",
|
|
315
|
+
"--entrypoint",
|
|
316
|
+
"sh",
|
|
317
|
+
"caelo-admin",
|
|
318
|
+
"-c",
|
|
319
|
+
`set -e; find ${dirs} -mindepth 1 -delete; tar -C / -xf -`,
|
|
320
|
+
),
|
|
321
|
+
{ stdinPath: join(work, FILES_TAR) },
|
|
322
|
+
);
|
|
323
|
+
if (replaced.exitCode !== 0) {
|
|
324
|
+
return changed(`Could not restore the site + media files: ${tail(replaced.stderr)}`);
|
|
325
|
+
}
|
|
326
|
+
|
|
327
|
+
log("Starting everything …");
|
|
328
|
+
const up = await runner.run(compose(caeloDir, "up", "-d", "--remove-orphans"));
|
|
329
|
+
if (up.exitCode !== 0) return changed(`docker compose up failed: ${tail(up.stderr)}`);
|
|
330
|
+
for (const service of ["admin", "gateway"] as const) {
|
|
331
|
+
const healthy = await waitHealthy(service, caeloDir, {
|
|
332
|
+
runner,
|
|
333
|
+
...(deps.sleep ? { sleep: deps.sleep } : {}),
|
|
334
|
+
...(deps.healthAttempts !== undefined ? { healthAttempts: deps.healthAttempts } : {}),
|
|
335
|
+
...(deps.healthIntervalMs !== undefined ? { healthIntervalMs: deps.healthIntervalMs } : {}),
|
|
336
|
+
});
|
|
337
|
+
if (!healthy.ok)
|
|
338
|
+
return changed(`The ${service} is not healthy after the restore: ${healthy.error}`);
|
|
339
|
+
}
|
|
340
|
+
const caddy = await deps.refreshCaddy();
|
|
341
|
+
if (!caddy.ok)
|
|
342
|
+
return changed(`Could not reload Caddy with the restored domains: ${caddy.error}`);
|
|
343
|
+
return { ok: true, manifest: checked.manifest };
|
|
344
|
+
} finally {
|
|
345
|
+
rmSync(work, { recursive: true, force: true });
|
|
346
|
+
}
|
|
347
|
+
}
|
|
348
|
+
|
|
349
|
+
/** Whether an unpacked archive has every part a restore needs, in a format this CLI reads. */
|
|
350
|
+
export function checkArchive(
|
|
351
|
+
dir: string,
|
|
352
|
+
): { ok: true; manifest: BackupManifest } | { ok: false; error: string } {
|
|
353
|
+
const manifestPath = join(dir, MANIFEST);
|
|
354
|
+
if (!existsSync(manifestPath)) {
|
|
355
|
+
return { ok: false, error: "The archive has no manifest.json — it is not a Caelo backup." };
|
|
356
|
+
}
|
|
357
|
+
let manifest: BackupManifest;
|
|
358
|
+
try {
|
|
359
|
+
manifest = JSON.parse(readFileSync(manifestPath, "utf8")) as BackupManifest;
|
|
360
|
+
} catch (e) {
|
|
361
|
+
return { ok: false, error: `manifest.json is not JSON: ${(e as Error).message}` };
|
|
362
|
+
}
|
|
363
|
+
if (manifest.format !== BACKUP_FORMAT) {
|
|
364
|
+
return {
|
|
365
|
+
ok: false,
|
|
366
|
+
error: `The backup has format ${manifest.format}; this provisioner restores format ${BACKUP_FORMAT}. Use the provisioner of Caelo ${manifest.caeloVersion}.`,
|
|
367
|
+
};
|
|
368
|
+
}
|
|
369
|
+
for (const part of [
|
|
370
|
+
join(DB_DIR, DATABASE_DUMPS.cms_admin),
|
|
371
|
+
join(DB_DIR, DATABASE_DUMPS.cms_public),
|
|
372
|
+
FILES_TAR,
|
|
373
|
+
]) {
|
|
374
|
+
if (!existsSync(join(dir, part)) || statSync(join(dir, part)).size === 0) {
|
|
375
|
+
return { ok: false, error: `The archive is incomplete: ${part} is missing or empty.` };
|
|
376
|
+
}
|
|
377
|
+
}
|
|
378
|
+
return { ok: true, manifest };
|
|
379
|
+
}
|
|
380
|
+
|
|
381
|
+
/** The members a backup archive may hold — anything else is refused before extraction. */
|
|
382
|
+
const ARCHIVE_MEMBERS = new Set([
|
|
383
|
+
"",
|
|
384
|
+
MANIFEST,
|
|
385
|
+
DB_DIR,
|
|
386
|
+
...Object.values(DATABASE_DUMPS).map((f) => `${DB_DIR}/${f}`),
|
|
387
|
+
FILES_TAR,
|
|
388
|
+
]);
|
|
389
|
+
|
|
390
|
+
/**
|
|
391
|
+
* Whether the .tar.gz at `path` holds only the members a backup writes, as
|
|
392
|
+
* plain files and directories — checked before `tar -x` writes anything.
|
|
393
|
+
*/
|
|
394
|
+
export function checkArchiveMembers(path: string): { ok: true } | { ok: false; error: string } {
|
|
395
|
+
let entries: TarEntry[];
|
|
396
|
+
try {
|
|
397
|
+
entries = listTarEntries(gunzipSync(readFileSync(path)));
|
|
398
|
+
} catch (e) {
|
|
399
|
+
return {
|
|
400
|
+
ok: false,
|
|
401
|
+
error: `${path} is not a readable backup archive (${(e as Error).message}).`,
|
|
402
|
+
};
|
|
403
|
+
}
|
|
404
|
+
for (const e of entries) {
|
|
405
|
+
const name = normalizeMemberName(e.name);
|
|
406
|
+
if (escapesRoot(e.name) || !ARCHIVE_MEMBERS.has(name)) {
|
|
407
|
+
return {
|
|
408
|
+
ok: false,
|
|
409
|
+
error: `The archive holds an unexpected member ${JSON.stringify(e.name)} — it is not a Caelo backup.`,
|
|
410
|
+
};
|
|
411
|
+
}
|
|
412
|
+
if (e.type !== (name === "" || name === DB_DIR ? "directory" : "file")) {
|
|
413
|
+
return {
|
|
414
|
+
ok: false,
|
|
415
|
+
error: `The archive member ${JSON.stringify(e.name)} is a ${e.type}, not what a Caelo backup writes.`,
|
|
416
|
+
};
|
|
417
|
+
}
|
|
418
|
+
}
|
|
419
|
+
return { ok: true };
|
|
420
|
+
}
|
|
421
|
+
|
|
422
|
+
/**
|
|
423
|
+
* Whether files.tar only writes below the site and media directories:
|
|
424
|
+
* relative paths without `..`, files / directories / relative symlinks
|
|
425
|
+
* inside the tree, and no member below a symlink (which would write
|
|
426
|
+
* wherever the link points).
|
|
427
|
+
*/
|
|
428
|
+
export function checkFilesTar(path: string): { ok: true } | { ok: false; error: string } {
|
|
429
|
+
let entries: TarEntry[];
|
|
430
|
+
try {
|
|
431
|
+
entries = listTarEntries(readFileSync(path));
|
|
432
|
+
} catch (e) {
|
|
433
|
+
return { ok: false, error: `files.tar is not readable (${(e as Error).message}).` };
|
|
434
|
+
}
|
|
435
|
+
const symlinks: string[] = [];
|
|
436
|
+
const refuse = (e: TarEntry, why: string) => ({
|
|
437
|
+
ok: false as const,
|
|
438
|
+
error: `files.tar member ${JSON.stringify(e.name)} ${why} — the archive is not a Caelo backup.`,
|
|
439
|
+
});
|
|
440
|
+
for (const e of entries) {
|
|
441
|
+
const name = normalizeMemberName(e.name);
|
|
442
|
+
if (escapesRoot(e.name)) return refuse(e, "points outside the content directories");
|
|
443
|
+
if (!CONTENT_PATHS.some((root) => name === root || name.startsWith(`${root}/`))) {
|
|
444
|
+
return refuse(e, "is outside the site and media directories");
|
|
445
|
+
}
|
|
446
|
+
if (symlinks.some((link) => name.startsWith(`${link}/`)))
|
|
447
|
+
return refuse(e, "lies below a symlink");
|
|
448
|
+
if (e.type === "symlink") {
|
|
449
|
+
const target = e.linkTarget ?? "";
|
|
450
|
+
if (target === "" || escapesRoot(target)) return refuse(e, "links outside its directory");
|
|
451
|
+
symlinks.push(name);
|
|
452
|
+
} else if (e.type === "other") {
|
|
453
|
+
return refuse(e, "is not a file, directory or symlink");
|
|
454
|
+
}
|
|
455
|
+
}
|
|
456
|
+
return { ok: true };
|
|
457
|
+
}
|
|
458
|
+
|
|
459
|
+
/** A scratch dir under `.caelo/backups/` (mode 700) — same filesystem, never world-readable. */
|
|
460
|
+
function workDir(caeloDir: string, kind: "backup" | "restore", at: Date): string {
|
|
461
|
+
const dir = join(caeloDir, "backups", `.${kind}-${at.toISOString().replace(/[:.]/g, "-")}`);
|
|
462
|
+
mkdirSync(dir, { recursive: true, mode: 0o700 });
|
|
463
|
+
return dir;
|
|
464
|
+
}
|
|
465
|
+
|
|
466
|
+
function unchanged(error: string): RestoreResult {
|
|
467
|
+
return {
|
|
468
|
+
ok: false,
|
|
469
|
+
error: `${error} Nothing on the install was changed.`,
|
|
470
|
+
installChanged: false,
|
|
471
|
+
};
|
|
472
|
+
}
|
|
473
|
+
|
|
474
|
+
function changed(error: string): RestoreResult {
|
|
475
|
+
return { ok: false, error, installChanged: true };
|
|
476
|
+
}
|
|
477
|
+
|
|
478
|
+
function tail(s: string): string {
|
|
479
|
+
return s.trim().split("\n").slice(-5).join("\n").slice(0, 800);
|
|
480
|
+
}
|
|
@@ -2,7 +2,12 @@
|
|
|
2
2
|
|
|
3
3
|
import { describe, expect, it } from "bun:test";
|
|
4
4
|
import {
|
|
5
|
+
CADDY_IMAGE,
|
|
6
|
+
COMPOSE_ADMIN_UPSTREAM,
|
|
7
|
+
COMPOSE_APP_SERVICES,
|
|
8
|
+
COMPOSE_GATEWAY_UPSTREAM,
|
|
5
9
|
type ComposeSpec,
|
|
10
|
+
composeProjectName,
|
|
6
11
|
generateDockerCompose,
|
|
7
12
|
ghcrImageDigest,
|
|
8
13
|
ghcrImageRef,
|
|
@@ -15,11 +20,11 @@ const base: Omit<ComposeSpec, "images"> = {
|
|
|
15
20
|
domain: "example.com",
|
|
16
21
|
postgresPassword: "pgpw",
|
|
17
22
|
rolePasswords: { admin: "adminpw", public: "publicpw", gateway: "gatewaypw" },
|
|
18
|
-
minioRootUser: "caelo",
|
|
19
|
-
minioRootPassword: "miniopw",
|
|
20
23
|
caeloSecretKek: "0".repeat(64),
|
|
21
24
|
anthropicApiKey: "sk-ant-x",
|
|
22
25
|
diskSize: "20Gi",
|
|
26
|
+
ports: { http: 80, https: 443 },
|
|
27
|
+
project: "caelo-example-com",
|
|
23
28
|
};
|
|
24
29
|
|
|
25
30
|
/** The block of one Compose service, up to the next blank line. */
|
|
@@ -77,11 +82,90 @@ describe("self-hosted Compose — release images (#696)", () => {
|
|
|
77
82
|
expect(gateway).not.toMatch(/ADMIN_DATABASE|adminpw|CAELO_SECRET_KEK/);
|
|
78
83
|
});
|
|
79
84
|
|
|
80
|
-
it("
|
|
81
|
-
expect(service(yaml, "caddy")).toContain(
|
|
85
|
+
it("runs Caddy on the pinned image the Caddyfile is validated against, Postgres as before", () => {
|
|
86
|
+
expect(service(yaml, "caddy")).toContain(`image: ${CADDY_IMAGE}`);
|
|
87
|
+
expect(CADDY_IMAGE).toMatch(/^caddy:\d+\.\d+\.\d+-alpine$/);
|
|
82
88
|
expect(service(yaml, "postgres")).toContain("image: postgres:16-alpine");
|
|
83
89
|
});
|
|
84
90
|
|
|
91
|
+
it("mounts the bootstrap script Postgres runs on its first start", () => {
|
|
92
|
+
expect(service(yaml, "postgres")).toContain(
|
|
93
|
+
"- ./bootstrap.sh:/docker-entrypoint-initdb.d/00_bootstrap.sh:ro",
|
|
94
|
+
);
|
|
95
|
+
});
|
|
96
|
+
|
|
97
|
+
it("has no pgBackRest sidecar and bind-mounts only paths the CLI creates (#707)", () => {
|
|
98
|
+
expect(yaml).not.toContain("pgbackrest");
|
|
99
|
+
expect(yaml).not.toContain("caelo-pg-backups");
|
|
100
|
+
// Each is written (files) or created (dirs) by `init` / `up`; a path
|
|
101
|
+
// nothing creates becomes an empty root-owned dir under Docker.
|
|
102
|
+
const hostPaths = new Set([...yaml.matchAll(/- \.\/([\w.-]+):/g)].map((m) => m[1]));
|
|
103
|
+
expect([...hostPaths].sort()).toEqual([
|
|
104
|
+
"Caddyfile",
|
|
105
|
+
"bootstrap",
|
|
106
|
+
"bootstrap.sh",
|
|
107
|
+
"media",
|
|
108
|
+
"output",
|
|
109
|
+
]);
|
|
110
|
+
});
|
|
111
|
+
|
|
112
|
+
it("names no containers, so an install can share a host with other stacks (#707)", () => {
|
|
113
|
+
expect(yaml).not.toContain("container_name");
|
|
114
|
+
});
|
|
115
|
+
|
|
116
|
+
it("defines every service restore stops and Caddy proxies to (#707)", () => {
|
|
117
|
+
for (const name of COMPOSE_APP_SERVICES) expect(yaml).toContain(`\n ${name}:\n`);
|
|
118
|
+
expect(COMPOSE_ADMIN_UPSTREAM).toBe("caelo-admin:5173");
|
|
119
|
+
expect(COMPOSE_GATEWAY_UPSTREAM).toBe("caelo-gateway:8090");
|
|
120
|
+
expect(service(yaml, "caelo-admin")).toContain('PORT: "5173"');
|
|
121
|
+
expect(service(yaml, "caelo-gateway")).toContain('GATEWAY_PORT: "8090"');
|
|
122
|
+
});
|
|
123
|
+
|
|
124
|
+
it("names its Compose project, so two installs on one host never share containers or volumes", () => {
|
|
125
|
+
expect(yaml).toContain("\nname: caelo-example-com\n");
|
|
126
|
+
expect(composeProjectName("Shop.Example.co.uk")).toBe("caelo-shop-example-co-uk");
|
|
127
|
+
expect(() =>
|
|
128
|
+
generateDockerCompose({
|
|
129
|
+
...base,
|
|
130
|
+
project: "Bad Name",
|
|
131
|
+
images: { mode: "release", admin: ADMIN, gateway: GATEWAY },
|
|
132
|
+
}),
|
|
133
|
+
).toThrow(/not a Compose project name/);
|
|
134
|
+
});
|
|
135
|
+
|
|
136
|
+
it("puts a non-443 HTTPS port into the admin's site URL", () => {
|
|
137
|
+
expect(service(yaml, "caelo-admin")).toContain('CAELO_SITE_URL: "https://example.com"');
|
|
138
|
+
const ported = generateDockerCompose({
|
|
139
|
+
...base,
|
|
140
|
+
ports: { http: 18080, https: 18443 },
|
|
141
|
+
images: { mode: "release", admin: ADMIN, gateway: GATEWAY },
|
|
142
|
+
});
|
|
143
|
+
expect(service(ported, "caelo-admin")).toContain('CAELO_SITE_URL: "https://example.com:18443"');
|
|
144
|
+
});
|
|
145
|
+
|
|
146
|
+
it("publishes Caddy on the configured host ports", () => {
|
|
147
|
+
expect(service(yaml, "caddy")).toContain('ports: ["80:80", "443:443"]');
|
|
148
|
+
const custom = generateDockerCompose({
|
|
149
|
+
...base,
|
|
150
|
+
ports: { http: 18080, https: 18443 },
|
|
151
|
+
images: { mode: "release", admin: ADMIN, gateway: GATEWAY },
|
|
152
|
+
});
|
|
153
|
+
expect(service(custom, "caddy")).toContain('ports: ["18080:80", "18443:443"]');
|
|
154
|
+
for (const ports of [
|
|
155
|
+
{ http: 0, https: 443 },
|
|
156
|
+
{ http: 80, https: 70000 },
|
|
157
|
+
{ http: 8080, https: 8080 },
|
|
158
|
+
]) {
|
|
159
|
+
expect(() =>
|
|
160
|
+
generateDockerCompose({
|
|
161
|
+
...base,
|
|
162
|
+
ports,
|
|
163
|
+
images: { mode: "release", admin: ADMIN, gateway: GATEWAY },
|
|
164
|
+
}),
|
|
165
|
+
).toThrow();
|
|
166
|
+
}
|
|
167
|
+
});
|
|
168
|
+
|
|
85
169
|
it("refuses a tag, a foreign repository or a malformed digest", () => {
|
|
86
170
|
for (const admin of [
|
|
87
171
|
"ghcr.io/caelo-cms/admin:latest",
|
|
@@ -45,8 +45,20 @@ export interface HostRunner {
|
|
|
45
45
|
/**
|
|
46
46
|
* Run `argv` to completion. With `stdoutPath`, stdout streams into that
|
|
47
47
|
* file (mode 600) instead of being captured — used for database dumps.
|
|
48
|
+
* With `stdinPath`, that file is the process's stdin (restores). With
|
|
49
|
+
* `inherit`, stdout + stderr go to the terminal (long pulls and starts
|
|
50
|
+
* the operator should watch) and come back empty.
|
|
48
51
|
*/
|
|
49
|
-
run(argv: readonly string[], opts?:
|
|
52
|
+
run(argv: readonly string[], opts?: HostRunOptions): Promise<CommandResult>;
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
/** See {@link HostRunner.run}. */
|
|
56
|
+
export interface HostRunOptions {
|
|
57
|
+
/** Extra environment variables on top of the CLI's own. */
|
|
58
|
+
readonly env?: Readonly<Record<string, string>>;
|
|
59
|
+
readonly stdoutPath?: string;
|
|
60
|
+
readonly stdinPath?: string;
|
|
61
|
+
readonly inherit?: boolean;
|
|
50
62
|
}
|
|
51
63
|
|
|
52
64
|
/** The real runner: Bun.spawn; a missing binary is exit code 127, not a throw. */
|
|
@@ -54,13 +66,16 @@ export const bunHostRunner: HostRunner = {
|
|
|
54
66
|
async run(argv, opts) {
|
|
55
67
|
try {
|
|
56
68
|
const proc = Bun.spawn([...argv], {
|
|
57
|
-
|
|
58
|
-
|
|
69
|
+
...(opts?.env ? { env: { ...process.env, ...opts.env } } : {}),
|
|
70
|
+
stdin: opts?.stdinPath ? Bun.file(opts.stdinPath) : "ignore",
|
|
71
|
+
stdout: opts?.stdoutPath ? Bun.file(opts.stdoutPath) : opts?.inherit ? "inherit" : "pipe",
|
|
72
|
+
stderr: opts?.inherit ? "inherit" : "pipe",
|
|
59
73
|
});
|
|
60
|
-
const stdout =
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
74
|
+
const stdout =
|
|
75
|
+
opts?.stdoutPath || opts?.inherit
|
|
76
|
+
? ""
|
|
77
|
+
: await new Response(proc.stdout as ReadableStream).text();
|
|
78
|
+
const stderr = opts?.inherit ? "" : await new Response(proc.stderr as ReadableStream).text();
|
|
64
79
|
return { exitCode: await proc.exited, stdout, stderr };
|
|
65
80
|
} catch (e) {
|
|
66
81
|
return { exitCode: 127, stdout: "", stderr: e instanceof Error ? e.message : String(e) };
|