@caelo-cms/provisioning 0.10.48 → 0.10.50
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/caddy/Caddyfile.production +5 -1
- package/caddy/Caddyfile.staging +5 -1
- package/dist/agent-db.js +102 -58
- package/dist/caddy-redirects.d.ts +15 -0
- package/dist/caddy-redirects.d.ts.map +1 -0
- package/dist/caddy-redirects.js +65 -0
- package/dist/caddy-redirects.js.map +1 -0
- package/dist/caddy.d.ts +22 -2
- package/dist/caddy.d.ts.map +1 -1
- package/dist/caddy.js +66 -17
- package/dist/caddy.js.map +1 -1
- package/dist/cli-routing.d.ts.map +1 -1
- package/dist/cli-routing.js +7 -0
- package/dist/cli-routing.js.map +1 -1
- package/dist/cli.js +248 -254
- package/dist/cli.js.map +1 -1
- package/dist/compose.d.ts +54 -2
- package/dist/compose.d.ts.map +1 -1
- package/dist/compose.js +86 -32
- package/dist/compose.js.map +1 -1
- package/dist/gcp-names.d.ts +12 -0
- package/dist/gcp-names.d.ts.map +1 -1
- package/dist/gcp-names.js +21 -0
- package/dist/gcp-names.js.map +1 -1
- package/dist/gcp-url-map.d.ts +27 -0
- package/dist/gcp-url-map.d.ts.map +1 -0
- package/dist/gcp-url-map.js +62 -0
- package/dist/gcp-url-map.js.map +1 -0
- package/dist/index.d.ts +0 -1
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +0 -1
- package/dist/index.js.map +1 -1
- package/dist/infra-contract.d.ts +20 -1
- package/dist/infra-contract.d.ts.map +1 -1
- package/dist/infra-contract.js +39 -1
- package/dist/infra-contract.js.map +1 -1
- package/dist/public-role-isolation.d.ts +19 -0
- package/dist/public-role-isolation.d.ts.map +1 -0
- package/dist/public-role-isolation.js +69 -0
- package/dist/public-role-isolation.js.map +1 -0
- package/dist/release-contract.d.ts.map +1 -1
- package/dist/release-contract.js +3 -0
- package/dist/release-contract.js.map +1 -1
- package/dist/release-manifest.d.ts +2 -41
- package/dist/release-manifest.d.ts.map +1 -1
- package/dist/release-manifest.js +20 -122
- package/dist/release-manifest.js.map +1 -1
- package/dist/self-hosted-backup.d.ts +86 -0
- package/dist/self-hosted-backup.d.ts.map +1 -0
- package/dist/self-hosted-backup.js +350 -0
- package/dist/self-hosted-backup.js.map +1 -0
- package/dist/self-hosted-release.d.ts +12 -3
- package/dist/self-hosted-release.d.ts.map +1 -1
- package/dist/self-hosted-release.js +6 -4
- package/dist/self-hosted-release.js.map +1 -1
- package/dist/self-hosted-start.d.ts +50 -0
- package/dist/self-hosted-start.d.ts.map +1 -0
- package/dist/self-hosted-start.js +59 -0
- package/dist/self-hosted-start.js.map +1 -0
- package/dist/self-hosted-upgrade.d.ts +30 -0
- package/dist/self-hosted-upgrade.d.ts.map +1 -1
- package/dist/self-hosted-upgrade.js +29 -7
- package/dist/self-hosted-upgrade.js.map +1 -1
- package/dist/tar-entries.d.ts +14 -0
- package/dist/tar-entries.d.ts.map +1 -0
- package/dist/tar-entries.js +104 -0
- package/dist/tar-entries.js.map +1 -0
- package/dist/upgrade/converge.d.ts +2 -2
- package/dist/upgrade/converge.d.ts.map +1 -1
- package/dist/upgrade/converge.js +44 -2
- package/dist/upgrade/converge.js.map +1 -1
- package/dist/upgrade/release-images.d.ts +14 -2
- package/dist/upgrade/release-images.d.ts.map +1 -1
- package/dist/upgrade/release-images.js +4 -2
- package/dist/upgrade/release-images.js.map +1 -1
- package/dist/upgrader-assess.d.ts +4 -1
- package/dist/upgrader-assess.d.ts.map +1 -1
- package/dist/upgrader-assess.js +8 -1
- package/dist/upgrader-assess.js.map +1 -1
- package/dist/upgrader-job.d.ts +3 -1
- package/dist/upgrader-job.d.ts.map +1 -1
- package/dist/upgrader-job.js.map +1 -1
- package/dist/url-map-redirects-access.d.ts +15 -0
- package/dist/url-map-redirects-access.d.ts.map +1 -0
- package/dist/url-map-redirects-access.js +81 -0
- package/dist/url-map-redirects-access.js.map +1 -0
- package/dist/wizard.d.ts.map +1 -1
- package/dist/wizard.js +3 -1
- package/dist/wizard.js.map +1 -1
- package/dist/wizards/gcp.d.ts.map +1 -1
- package/dist/wizards/gcp.js +45 -0
- package/dist/wizards/gcp.js.map +1 -1
- package/package.json +2 -2
- package/src/admin-memory.test.ts +2 -2
- package/src/agent/run.test.ts +55 -0
- package/src/caddy-redirects.docker.test.ts +255 -0
- package/src/caddy-redirects.test.ts +115 -0
- package/src/caddy-redirects.ts +68 -0
- package/src/caddy-validate.test.ts +119 -0
- package/src/caddy.test.ts +108 -0
- package/src/caddy.ts +89 -20
- package/src/cli-routing.test.ts +12 -0
- package/src/cli-routing.ts +7 -0
- package/src/cli.ts +299 -287
- package/src/compose.ts +106 -34
- package/src/gcp-names.ts +23 -0
- package/src/gcp-url-map.test.ts +150 -0
- package/src/gcp-url-map.ts +81 -0
- package/src/index.test.ts +18 -146
- package/src/index.ts +0 -9
- package/src/infra-contract.test.ts +39 -0
- package/src/infra-contract.ts +42 -1
- package/src/public-role-isolation.test.ts +89 -0
- package/src/public-role-isolation.ts +83 -0
- package/src/release-contract.ts +3 -0
- package/src/release-manifest.test.ts +20 -1
- package/src/release-manifest.ts +39 -158
- package/src/self-hosted-backup.test.ts +344 -0
- package/src/self-hosted-backup.ts +480 -0
- package/src/self-hosted-compose.test.ts +88 -4
- package/src/self-hosted-release.ts +22 -7
- package/src/self-hosted-start.test.ts +68 -0
- package/src/self-hosted-start.ts +123 -0
- package/src/self-hosted-upgrade.ts +35 -8
- package/src/stack-contract.test.ts +2 -2
- package/src/tar-entries.test.ts +52 -0
- package/src/tar-entries.ts +116 -0
- package/src/upgrade/converge.ts +51 -4
- package/src/upgrade/release-images.ts +4 -2
- package/src/upgrade/upgrade.test.ts +78 -1
- package/src/upgrader-assess.test.ts +64 -1
- package/src/upgrader-assess.ts +20 -2
- package/src/upgrader-job.test.ts +3 -0
- package/src/upgrader-job.ts +3 -0
- package/src/url-map-redirects-access.test.ts +88 -0
- package/src/url-map-redirects-access.ts +89 -0
- package/src/wizard.ts +3 -1
- package/src/wizards/gcp.ts +48 -0
- package/stacks/aws/index.ts +34 -2
- package/stacks/azure/index.ts +13 -0
- package/stacks/gcp/index.ts +24 -17
- package/stacks/gcp-firebase/index.ts +7 -0
- package/stacks/self-hosted/Pulumi.yaml +2 -2
- package/stacks/self-hosted/README.md +3 -3
- package/stacks/self-hosted/index.ts +45 -22
- package/dist/redirects-emit.d.ts +0 -52
- package/dist/redirects-emit.d.ts.map +0 -1
- package/dist/redirects-emit.js +0 -111
- package/dist/redirects-emit.js.map +0 -1
- package/src/redirects-emit.ts +0 -175
|
@@ -0,0 +1,68 @@
|
|
|
1
|
+
// SPDX-License-Identifier: MPL-2.0
|
|
2
|
+
|
|
3
|
+
import { describe, expect, it } from "bun:test";
|
|
4
|
+
import type { CommandResult, HostRunner } from "./self-hosted-release.js";
|
|
5
|
+
import { startSelfHosted } from "./self-hosted-start.js";
|
|
6
|
+
|
|
7
|
+
const DIR = "/srv/install/.caelo";
|
|
8
|
+
|
|
9
|
+
/** A runner that records every command (minus the `docker compose -f <file>` prefix). */
|
|
10
|
+
function fakeRunner(fail?: (cmd: string) => boolean): HostRunner & { calls: string[] } {
|
|
11
|
+
const calls: string[] = [];
|
|
12
|
+
return {
|
|
13
|
+
calls,
|
|
14
|
+
async run(argv): Promise<CommandResult> {
|
|
15
|
+
const cmd = argv.join(" ").replace(`docker compose -f ${DIR}/docker-compose.yml `, "");
|
|
16
|
+
calls.push(cmd);
|
|
17
|
+
return { exitCode: fail?.(cmd) ? 1 : 0, stdout: "", stderr: fail?.(cmd) ? "boom" : "" };
|
|
18
|
+
},
|
|
19
|
+
};
|
|
20
|
+
}
|
|
21
|
+
|
|
22
|
+
const MIGRATE =
|
|
23
|
+
"run --rm --no-deps -T caelo-admin bun --bun /app/packages/migrations/src/migrate.ts";
|
|
24
|
+
|
|
25
|
+
describe("startSelfHosted (#707 — one command ends with a serving install)", () => {
|
|
26
|
+
it("pulls, starts Postgres, migrates, starts everything, then health-checks both services", async () => {
|
|
27
|
+
const runner = fakeRunner();
|
|
28
|
+
const r = await startSelfHosted({ caeloDir: DIR, runner, log: () => {} });
|
|
29
|
+
expect(r).toEqual({ ok: true });
|
|
30
|
+
expect(runner.calls.map((c) => c.replace(/ -e .*/, " -e <probe>"))).toEqual([
|
|
31
|
+
"pull",
|
|
32
|
+
"up -d --wait postgres",
|
|
33
|
+
`${MIGRATE} admin`,
|
|
34
|
+
`${MIGRATE} public`,
|
|
35
|
+
"up -d --remove-orphans",
|
|
36
|
+
"exec -T caelo-admin bun -e <probe>",
|
|
37
|
+
"exec -T caelo-gateway bun -e <probe>",
|
|
38
|
+
"exec -T caddy caddy reload --config /etc/caddy/Caddyfile --adapter caddyfile",
|
|
39
|
+
]);
|
|
40
|
+
});
|
|
41
|
+
|
|
42
|
+
it("stops at a failed migration — nothing else is started", async () => {
|
|
43
|
+
const runner = fakeRunner((c) => c.endsWith("migrate.ts admin"));
|
|
44
|
+
const r = await startSelfHosted({ caeloDir: DIR, runner, log: () => {} });
|
|
45
|
+
expect(r.ok).toBe(false);
|
|
46
|
+
if (!r.ok) {
|
|
47
|
+
expect(r.stage).toBe("migrate");
|
|
48
|
+
expect(r.error).toContain("admin migrations failed");
|
|
49
|
+
}
|
|
50
|
+
expect(runner.calls).not.toContain("up -d --remove-orphans");
|
|
51
|
+
});
|
|
52
|
+
|
|
53
|
+
it("reports an unhealthy admin with where to look", async () => {
|
|
54
|
+
const runner = fakeRunner((c) => c.startsWith("exec -T caelo-admin"));
|
|
55
|
+
const r = await startSelfHosted({
|
|
56
|
+
caeloDir: DIR,
|
|
57
|
+
runner,
|
|
58
|
+
log: () => {},
|
|
59
|
+
healthAttempts: 2,
|
|
60
|
+
sleep: async () => {},
|
|
61
|
+
});
|
|
62
|
+
expect(r.ok).toBe(false);
|
|
63
|
+
if (!r.ok) {
|
|
64
|
+
expect(r.stage).toBe("health");
|
|
65
|
+
expect(r.error).toContain("logs caelo-admin");
|
|
66
|
+
}
|
|
67
|
+
});
|
|
68
|
+
});
|
|
@@ -0,0 +1,123 @@
|
|
|
1
|
+
// SPDX-License-Identifier: MPL-2.0
|
|
2
|
+
|
|
3
|
+
/**
|
|
4
|
+
* Start (or converge) a self-hosted Compose install — what `init` and `up`
|
|
5
|
+
* run after writing `.caelo/`, so one command ends with a serving install
|
|
6
|
+
* (CLAUDE.md §11.C, issue #707) instead of printing steps to copy.
|
|
7
|
+
*
|
|
8
|
+
* Order, and why:
|
|
9
|
+
* 1. pull every image, with the progress on the terminal (the admin image
|
|
10
|
+
* is large; a silent multi-minute step reads as a hang);
|
|
11
|
+
* 2. start Postgres and wait for its health check — on a fresh volume its
|
|
12
|
+
* first start runs bootstrap.sh (roles + databases);
|
|
13
|
+
* 3. apply the migrations with the install's admin image;
|
|
14
|
+
* 4. start everything (`--remove-orphans` drops services an older Compose
|
|
15
|
+
* file had, e.g. the pgBackRest sidecar);
|
|
16
|
+
* 5. health-check the admin (`/_caelo/health` proves the DB pool and its
|
|
17
|
+
* role) and the gateway;
|
|
18
|
+
* 6. reload Caddy — Compose does not restart a running Caddy when only
|
|
19
|
+
* its bind-mounted Caddyfile changed, so a re-run `up` would otherwise
|
|
20
|
+
* keep serving the old vhosts.
|
|
21
|
+
* Every step converges, so a re-run after a failure picks up where it
|
|
22
|
+
* stopped.
|
|
23
|
+
*/
|
|
24
|
+
|
|
25
|
+
import type { HostRunner } from "./self-hosted-release.js";
|
|
26
|
+
import { compose, runMigrations, waitHealthy } from "./self-hosted-upgrade.js";
|
|
27
|
+
|
|
28
|
+
export interface StartDeps {
|
|
29
|
+
readonly caeloDir: string;
|
|
30
|
+
readonly runner: HostRunner;
|
|
31
|
+
readonly log: (message: string) => void;
|
|
32
|
+
/** Health probe attempts per service and the pause between them. */
|
|
33
|
+
readonly healthAttempts?: number;
|
|
34
|
+
readonly healthIntervalMs?: number;
|
|
35
|
+
readonly sleep?: (ms: number) => Promise<void>;
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
export type StartResult =
|
|
39
|
+
| { readonly ok: true }
|
|
40
|
+
| { readonly ok: false; readonly stage: StartStage; readonly error: string };
|
|
41
|
+
|
|
42
|
+
type StartStage = "pull" | "postgres" | "migrate" | "start" | "health" | "caddy";
|
|
43
|
+
|
|
44
|
+
/** Bring the install's Compose stack up and prove it serves. Never throws for an expected failure. */
|
|
45
|
+
export async function startSelfHosted(deps: StartDeps): Promise<StartResult> {
|
|
46
|
+
const { caeloDir, runner, log } = deps;
|
|
47
|
+
log("Pulling the images …");
|
|
48
|
+
const pull = await runner.run(compose(caeloDir, "pull"), { inherit: true });
|
|
49
|
+
if (pull.exitCode !== 0) {
|
|
50
|
+
return fail("pull", "docker compose pull failed (see above). Check the network and re-run.");
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
log("Starting Postgres …");
|
|
54
|
+
const pg = await runner.run(compose(caeloDir, "up", "-d", "--wait", "postgres"), {
|
|
55
|
+
inherit: true,
|
|
56
|
+
});
|
|
57
|
+
if (pg.exitCode !== 0) {
|
|
58
|
+
return fail(
|
|
59
|
+
"postgres",
|
|
60
|
+
`Postgres did not become healthy (see above; \`docker compose -f ${caeloDir}/docker-compose.yml logs postgres\`).`,
|
|
61
|
+
);
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
log("Applying the database migrations …");
|
|
65
|
+
const migrated = await runMigrations(caeloDir, runner);
|
|
66
|
+
if (!migrated.ok) return fail("migrate", migrated.error);
|
|
67
|
+
|
|
68
|
+
log("Starting Caddy, the admin and the gateway …");
|
|
69
|
+
const up = await runner.run(compose(caeloDir, "up", "-d", "--remove-orphans"), {
|
|
70
|
+
inherit: true,
|
|
71
|
+
});
|
|
72
|
+
if (up.exitCode !== 0) return fail("start", "docker compose up failed (see above).");
|
|
73
|
+
|
|
74
|
+
for (const service of ["admin", "gateway"] as const) {
|
|
75
|
+
log(`Waiting for the ${service} to answer its health check …`);
|
|
76
|
+
const healthy = await waitHealthy(service, caeloDir, {
|
|
77
|
+
runner,
|
|
78
|
+
...(deps.sleep ? { sleep: deps.sleep } : {}),
|
|
79
|
+
...(deps.healthAttempts !== undefined ? { healthAttempts: deps.healthAttempts } : {}),
|
|
80
|
+
...(deps.healthIntervalMs !== undefined ? { healthIntervalMs: deps.healthIntervalMs } : {}),
|
|
81
|
+
});
|
|
82
|
+
if (!healthy.ok) {
|
|
83
|
+
return fail(
|
|
84
|
+
"health",
|
|
85
|
+
`The ${service} is not healthy: ${healthy.error}. See \`docker compose -f ${caeloDir}/docker-compose.yml logs caelo-${service}\`.`,
|
|
86
|
+
);
|
|
87
|
+
}
|
|
88
|
+
}
|
|
89
|
+
log("Reloading Caddy …");
|
|
90
|
+
const reloaded = await reloadCaddy(caeloDir, runner);
|
|
91
|
+
if (!reloaded.ok) return fail("caddy", reloaded.error);
|
|
92
|
+
return { ok: true };
|
|
93
|
+
}
|
|
94
|
+
|
|
95
|
+
/** `caddy reload` in the running Caddy container with the current Caddyfile. */
|
|
96
|
+
export async function reloadCaddy(
|
|
97
|
+
caeloDir: string,
|
|
98
|
+
runner: HostRunner,
|
|
99
|
+
): Promise<{ ok: true } | { ok: false; error: string }> {
|
|
100
|
+
const r = await runner.run(
|
|
101
|
+
compose(
|
|
102
|
+
caeloDir,
|
|
103
|
+
"exec",
|
|
104
|
+
"-T",
|
|
105
|
+
"caddy",
|
|
106
|
+
"caddy",
|
|
107
|
+
"reload",
|
|
108
|
+
"--config",
|
|
109
|
+
"/etc/caddy/Caddyfile",
|
|
110
|
+
"--adapter",
|
|
111
|
+
"caddyfile",
|
|
112
|
+
),
|
|
113
|
+
);
|
|
114
|
+
if (r.exitCode === 0) return { ok: true };
|
|
115
|
+
return {
|
|
116
|
+
ok: false,
|
|
117
|
+
error: `caddy reload failed: ${r.stderr.trim().split("\n").slice(-3).join("\n")}`,
|
|
118
|
+
};
|
|
119
|
+
}
|
|
120
|
+
|
|
121
|
+
function fail(stage: StartStage, error: string): StartResult {
|
|
122
|
+
return { ok: false, stage, error };
|
|
123
|
+
}
|
|
@@ -196,7 +196,7 @@ export function composePath(caeloDir: string): string {
|
|
|
196
196
|
}
|
|
197
197
|
|
|
198
198
|
/** `docker compose -f <file>` — every Compose call targets the install's file explicitly. */
|
|
199
|
-
function compose(caeloDir: string, ...args: string[]): string[] {
|
|
199
|
+
export function compose(caeloDir: string, ...args: string[]): string[] {
|
|
200
200
|
return ["docker", "compose", "-f", composePath(caeloDir), ...args];
|
|
201
201
|
}
|
|
202
202
|
|
|
@@ -415,12 +415,35 @@ export async function backupDatabases(
|
|
|
415
415
|
};
|
|
416
416
|
}
|
|
417
417
|
const dir = join(caeloDir, "backups", `pre-upgrade-${at.toISOString().replace(/[:.]/g, "-")}`);
|
|
418
|
+
const dumped = await dumpDatabases(caeloDir, runner, dir);
|
|
419
|
+
if (!dumped.ok) return { ok: false, error: `${dumped.error} The upgrade did not start.` };
|
|
420
|
+
return { ok: true, dir };
|
|
421
|
+
}
|
|
422
|
+
|
|
423
|
+
/** The files {@link dumpDatabases} writes, and what each holds. */
|
|
424
|
+
export const DATABASE_DUMPS = {
|
|
425
|
+
/** The cluster's roles (reference only — restores re-create them from config.json). */
|
|
426
|
+
globals: "globals.sql",
|
|
427
|
+
cms_admin: "cms_admin.dump",
|
|
428
|
+
cms_public: "cms_public.dump",
|
|
429
|
+
} as const;
|
|
430
|
+
|
|
431
|
+
/**
|
|
432
|
+
* pg_dump both databases (custom format) plus the cluster's roles into
|
|
433
|
+
* `dir` (created mode 700) from the running Postgres container. Every dump
|
|
434
|
+
* must exit 0 and be non-empty.
|
|
435
|
+
*/
|
|
436
|
+
export async function dumpDatabases(
|
|
437
|
+
caeloDir: string,
|
|
438
|
+
runner: HostRunner,
|
|
439
|
+
dir: string,
|
|
440
|
+
): Promise<{ ok: true } | { ok: false; error: string }> {
|
|
418
441
|
mkdirSync(dir, { recursive: true, mode: 0o700 });
|
|
419
442
|
chmodSync(dir, 0o700);
|
|
420
443
|
const dumps: [string, string[]][] = [
|
|
421
|
-
[
|
|
422
|
-
[
|
|
423
|
-
[
|
|
444
|
+
[DATABASE_DUMPS.globals, ["pg_dumpall", "-U", "caelo", "--globals-only"]],
|
|
445
|
+
[DATABASE_DUMPS.cms_admin, ["pg_dump", "-U", "caelo", "-Fc", "cms_admin"]],
|
|
446
|
+
[DATABASE_DUMPS.cms_public, ["pg_dump", "-U", "caelo", "-Fc", "cms_public"]],
|
|
424
447
|
];
|
|
425
448
|
for (const [name, cmd] of dumps) {
|
|
426
449
|
const out = join(dir, name);
|
|
@@ -430,12 +453,12 @@ export async function backupDatabases(
|
|
|
430
453
|
if (r.exitCode !== 0 || fileSize(out) === 0) {
|
|
431
454
|
return {
|
|
432
455
|
ok: false,
|
|
433
|
-
error: `Backup failed (${cmd.join(" ")} → ${out}): ${tail(r.stderr) || "empty dump"}
|
|
456
|
+
error: `Backup failed (${cmd.join(" ")} → ${out}): ${tail(r.stderr) || "empty dump"}.`,
|
|
434
457
|
};
|
|
435
458
|
}
|
|
436
459
|
chmodSync(out, 0o600);
|
|
437
460
|
}
|
|
438
|
-
return { ok: true
|
|
461
|
+
return { ok: true };
|
|
439
462
|
}
|
|
440
463
|
|
|
441
464
|
/**
|
|
@@ -483,10 +506,14 @@ const HEALTH_PROBE: Record<"admin" | "gateway", string> = {
|
|
|
483
506
|
'const r=await fetch("http://127.0.0.1:"+(process.env.PORT||process.env.GATEWAY_PORT||"8090")+"/healthz").catch(()=>null);process.exit(r&&r.ok?0:1)',
|
|
484
507
|
};
|
|
485
508
|
|
|
486
|
-
|
|
509
|
+
/**
|
|
510
|
+
* Probe `service` inside its container until it answers healthy, or give up
|
|
511
|
+
* after `healthAttempts` tries (default 45, 2 s apart).
|
|
512
|
+
*/
|
|
513
|
+
export async function waitHealthy(
|
|
487
514
|
service: "admin" | "gateway",
|
|
488
515
|
caeloDir: string,
|
|
489
|
-
deps: SelfHostedUpgradeDeps,
|
|
516
|
+
deps: Pick<SelfHostedUpgradeDeps, "runner" | "sleep" | "healthAttempts" | "healthIntervalMs">,
|
|
490
517
|
): Promise<{ ok: true } | { ok: false; error: string }> {
|
|
491
518
|
const attempts = deps.healthAttempts ?? 45;
|
|
492
519
|
const interval = deps.healthIntervalMs ?? 2000;
|
|
@@ -311,10 +311,10 @@ describe("self-hosted compose", () => {
|
|
|
311
311
|
domain: "acme.com",
|
|
312
312
|
postgresPassword: "p",
|
|
313
313
|
rolePasswords: { admin: "a", public: "b", gateway: "g" },
|
|
314
|
-
minioRootUser: "u",
|
|
315
|
-
minioRootPassword: "m",
|
|
316
314
|
caeloSecretKek: "k",
|
|
317
315
|
diskSize: "10G",
|
|
316
|
+
ports: { http: 80, https: 443 },
|
|
317
|
+
project: "caelo-example-com",
|
|
318
318
|
images: {
|
|
319
319
|
mode: "release" as const,
|
|
320
320
|
admin: `ghcr.io/caelo-cms/admin@sha256:${"a".repeat(64)}`,
|
|
@@ -0,0 +1,52 @@
|
|
|
1
|
+
// SPDX-License-Identifier: MPL-2.0
|
|
2
|
+
|
|
3
|
+
import { afterEach, beforeEach, describe, expect, it } from "bun:test";
|
|
4
|
+
import { mkdirSync, mkdtempSync, readFileSync, rmSync, symlinkSync, writeFileSync } from "node:fs";
|
|
5
|
+
import { tmpdir } from "node:os";
|
|
6
|
+
import { join } from "node:path";
|
|
7
|
+
import { bunHostRunner } from "./self-hosted-release.js";
|
|
8
|
+
import { escapesRoot, listTarEntries, normalizeMemberName } from "./tar-entries.js";
|
|
9
|
+
|
|
10
|
+
let dir: string;
|
|
11
|
+
beforeEach(() => {
|
|
12
|
+
dir = mkdtempSync(join(tmpdir(), "caelo-tar-test-"));
|
|
13
|
+
});
|
|
14
|
+
afterEach(() => rmSync(dir, { recursive: true, force: true }));
|
|
15
|
+
|
|
16
|
+
describe("listTarEntries", () => {
|
|
17
|
+
it("reads files, directories, symlinks and long (>100 byte) names from the host's tar", async () => {
|
|
18
|
+
const root = join(dir, "src");
|
|
19
|
+
const long = `${"d".repeat(60)}/${"e".repeat(60)}/${"f".repeat(30)}.txt`;
|
|
20
|
+
mkdirSync(join(root, long, ".."), { recursive: true });
|
|
21
|
+
writeFileSync(join(root, long), "x");
|
|
22
|
+
writeFileSync(join(root, "ä-ü.txt"), "umlaut");
|
|
23
|
+
symlinkSync("target", join(root, "link"));
|
|
24
|
+
const out = join(dir, "a.tar");
|
|
25
|
+
const r = await bunHostRunner.run(["tar", "-cf", out, "-C", root, "."], {
|
|
26
|
+
env: { COPYFILE_DISABLE: "1" },
|
|
27
|
+
});
|
|
28
|
+
expect(r.exitCode).toBe(0);
|
|
29
|
+
const entries = listTarEntries(readFileSync(out));
|
|
30
|
+
const byName = new Map(entries.map((e) => [normalizeMemberName(e.name), e]));
|
|
31
|
+
expect(byName.get(long)?.type).toBe("file");
|
|
32
|
+
expect(byName.get("ä-ü.txt")?.type).toBe("file");
|
|
33
|
+
expect(byName.get("d".repeat(60))?.type).toBe("directory");
|
|
34
|
+
expect(byName.get("link")).toMatchObject({ type: "symlink", linkTarget: "target" });
|
|
35
|
+
});
|
|
36
|
+
|
|
37
|
+
it("refuses a truncated archive instead of skipping members", () => {
|
|
38
|
+
const header = new Uint8Array(512);
|
|
39
|
+
header.set(new TextEncoder().encode("a.txt"), 0);
|
|
40
|
+
header.set(new TextEncoder().encode("00000001000\0"), 124); // 512 bytes of data, none present
|
|
41
|
+
header[156] = "0".charCodeAt(0);
|
|
42
|
+
expect(() => listTarEntries(header)).toThrow(/truncated/);
|
|
43
|
+
});
|
|
44
|
+
});
|
|
45
|
+
|
|
46
|
+
describe("escapesRoot", () => {
|
|
47
|
+
it("flags absolute and parent-relative member names", () => {
|
|
48
|
+
expect(escapesRoot("/etc/passwd")).toBe(true);
|
|
49
|
+
expect(escapesRoot("srv/caelo/../../app")).toBe(true);
|
|
50
|
+
expect(escapesRoot("./srv/caelo/a..b")).toBe(false);
|
|
51
|
+
});
|
|
52
|
+
});
|
|
@@ -0,0 +1,116 @@
|
|
|
1
|
+
// SPDX-License-Identifier: MPL-2.0
|
|
2
|
+
|
|
3
|
+
/**
|
|
4
|
+
* Read the member list of a tar archive WITHOUT extracting it, so a restore
|
|
5
|
+
* can refuse an archive before `tar -x` writes anything (PR #755 review: an
|
|
6
|
+
* absolute or `../` member, or a member written through a symlink, would
|
|
7
|
+
* otherwise land outside the target directory).
|
|
8
|
+
*
|
|
9
|
+
* Parses ustar headers plus the two extensions GNU tar and bsdtar emit for
|
|
10
|
+
* long names: GNU `L`/`K` records and pax `x`/`g` records (`path=`,
|
|
11
|
+
* `linkpath=`). Anything it cannot read is an error, never a skipped member.
|
|
12
|
+
*/
|
|
13
|
+
|
|
14
|
+
const BLOCK = 512;
|
|
15
|
+
|
|
16
|
+
export type TarEntryType = "file" | "directory" | "symlink" | "other";
|
|
17
|
+
|
|
18
|
+
export interface TarEntry {
|
|
19
|
+
readonly name: string;
|
|
20
|
+
readonly type: TarEntryType;
|
|
21
|
+
/** The link target, for symlinks. */
|
|
22
|
+
readonly linkTarget?: string;
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
/** Every member of an (uncompressed) tar archive, in archive order. */
|
|
26
|
+
export function listTarEntries(tar: Uint8Array): TarEntry[] {
|
|
27
|
+
const entries: TarEntry[] = [];
|
|
28
|
+
let offset = 0;
|
|
29
|
+
let longName: string | null = null;
|
|
30
|
+
let longLink: string | null = null;
|
|
31
|
+
let paxPath: string | null = null;
|
|
32
|
+
let paxLink: string | null = null;
|
|
33
|
+
while (offset + BLOCK <= tar.length) {
|
|
34
|
+
const header = tar.subarray(offset, offset + BLOCK);
|
|
35
|
+
if (header.every((b) => b === 0)) break; // end-of-archive marker
|
|
36
|
+
const size = octal(header.subarray(124, 136));
|
|
37
|
+
const typeflag = String.fromCharCode(header[156] ?? 0);
|
|
38
|
+
const body = tar.subarray(offset + BLOCK, offset + BLOCK + size);
|
|
39
|
+
if (body.length < size) throw new Error("truncated tar archive");
|
|
40
|
+
offset += BLOCK + Math.ceil(size / BLOCK) * BLOCK;
|
|
41
|
+
|
|
42
|
+
if (typeflag === "L") {
|
|
43
|
+
longName = cString(body);
|
|
44
|
+
continue;
|
|
45
|
+
}
|
|
46
|
+
if (typeflag === "K") {
|
|
47
|
+
longLink = cString(body);
|
|
48
|
+
continue;
|
|
49
|
+
}
|
|
50
|
+
if (typeflag === "x" || typeflag === "g") {
|
|
51
|
+
const pax = paxRecords(body);
|
|
52
|
+
if (typeflag === "x") {
|
|
53
|
+
paxPath = pax.path ?? null;
|
|
54
|
+
paxLink = pax.linkpath ?? null;
|
|
55
|
+
}
|
|
56
|
+
continue;
|
|
57
|
+
}
|
|
58
|
+
const prefix = cString(header.subarray(345, 500));
|
|
59
|
+
const shortName = cString(header.subarray(0, 100));
|
|
60
|
+
const name = paxPath ?? longName ?? (prefix ? `${prefix}/${shortName}` : shortName);
|
|
61
|
+
const link = paxLink ?? longLink ?? cString(header.subarray(157, 257));
|
|
62
|
+
longName = longLink = paxPath = paxLink = null;
|
|
63
|
+
const type: TarEntryType =
|
|
64
|
+
typeflag === "0" || typeflag === "\0" || typeflag === "7"
|
|
65
|
+
? "file"
|
|
66
|
+
: typeflag === "5"
|
|
67
|
+
? "directory"
|
|
68
|
+
: typeflag === "2"
|
|
69
|
+
? "symlink"
|
|
70
|
+
: "other";
|
|
71
|
+
entries.push(type === "symlink" ? { name, type, linkTarget: link } : { name, type });
|
|
72
|
+
}
|
|
73
|
+
return entries;
|
|
74
|
+
}
|
|
75
|
+
|
|
76
|
+
/** `./a/b/` → `a/b`; the archive root (`./`) → "". */
|
|
77
|
+
export function normalizeMemberName(name: string): string {
|
|
78
|
+
return name.replace(/^(\.\/)+/, "").replace(/\/+$/, "");
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
/** Whether a member name could write outside the extraction directory. */
|
|
82
|
+
export function escapesRoot(name: string): boolean {
|
|
83
|
+
return name.startsWith("/") || name.split("/").some((part) => part === "..");
|
|
84
|
+
}
|
|
85
|
+
|
|
86
|
+
function octal(field: Uint8Array): number {
|
|
87
|
+
const text = cString(field).trim();
|
|
88
|
+
if (text === "") return 0;
|
|
89
|
+
if (!/^[0-7]+$/.test(text)) throw new Error(`unreadable tar header size ${JSON.stringify(text)}`);
|
|
90
|
+
return Number.parseInt(text, 8);
|
|
91
|
+
}
|
|
92
|
+
|
|
93
|
+
function cString(bytes: Uint8Array): string {
|
|
94
|
+
const end = bytes.indexOf(0);
|
|
95
|
+
return new TextDecoder().decode(end === -1 ? bytes : bytes.subarray(0, end));
|
|
96
|
+
}
|
|
97
|
+
|
|
98
|
+
/** pax records: `<len> <key>=<value>\n`, repeated; `<len>` counts bytes. */
|
|
99
|
+
function paxRecords(body: Uint8Array): Record<string, string> {
|
|
100
|
+
const out: Record<string, string> = {};
|
|
101
|
+
const decode = (b: Uint8Array) => new TextDecoder().decode(b);
|
|
102
|
+
let i = 0;
|
|
103
|
+
while (i < body.length && body[i] !== 0) {
|
|
104
|
+
const space = body.indexOf(0x20, i);
|
|
105
|
+
if (space === -1) throw new Error("unreadable pax header");
|
|
106
|
+
const len = Number.parseInt(decode(body.subarray(i, space)), 10);
|
|
107
|
+
if (!Number.isFinite(len) || len <= 0 || i + len > body.length) {
|
|
108
|
+
throw new Error("unreadable pax header");
|
|
109
|
+
}
|
|
110
|
+
const record = decode(body.subarray(space + 1, i + len - 1));
|
|
111
|
+
const eq = record.indexOf("=");
|
|
112
|
+
if (eq !== -1) out[record.slice(0, eq)] = record.slice(eq + 1);
|
|
113
|
+
i += len;
|
|
114
|
+
}
|
|
115
|
+
return out;
|
|
116
|
+
}
|
package/src/upgrade/converge.ts
CHANGED
|
@@ -5,8 +5,9 @@
|
|
|
5
5
|
* service accounts, secrets or other cloud resources so the install matches
|
|
6
6
|
* what the release's stacks declare — the gateway's run SA and the
|
|
7
7
|
* CLI-generated runtime secrets, the stack's IAM/CDN/firewall invariants,
|
|
8
|
-
* MCP access through IAP,
|
|
9
|
-
* the
|
|
8
|
+
* MCP access through IAP, public_role's Cloud SQL role memberships (#633),
|
|
9
|
+
* the gateway's and public_role's database logins, the operator-access sync
|
|
10
|
+
* job, and dropping bindings nothing reads any more.
|
|
10
11
|
*
|
|
11
12
|
* It needs the rights of whoever provisioned the install, so only
|
|
12
13
|
* `cms-provision upgrade` (the operator's own identity) runs it — never the
|
|
@@ -33,6 +34,7 @@ import { upgraderServiceAccountEmail } from "../gcp-names.js";
|
|
|
33
34
|
import type { InstallMetadata } from "../install-state.js";
|
|
34
35
|
import { ensureMcpIapAccess, type IapResource } from "../mcp-iap.js";
|
|
35
36
|
import { ensureOperatorAccessSync, resolveOperatorAccessTarget } from "../operator-access.js";
|
|
37
|
+
import { revokePublicRoleMemberships } from "../public-role-isolation.js";
|
|
36
38
|
import {
|
|
37
39
|
ensureGatewayServiceAccount,
|
|
38
40
|
ensureGeneratedSecrets,
|
|
@@ -52,6 +54,7 @@ import {
|
|
|
52
54
|
removeRetiredIamBindings,
|
|
53
55
|
} from "../stack-converge.js";
|
|
54
56
|
import { ensureUpgraderJob, resolveUpgraderTarget } from "../upgrader-setup.js";
|
|
57
|
+
import { ensureUrlMapRedirectsAccess } from "../url-map-redirects-access.js";
|
|
55
58
|
import type { AppRollHooks, AppRollPlan, RollPlan, RollSlug } from "./app-roll.js";
|
|
56
59
|
import type { UpgradeReporter } from "./reporter.js";
|
|
57
60
|
|
|
@@ -90,8 +93,8 @@ export interface ConvergeDeps {
|
|
|
90
93
|
|
|
91
94
|
export interface ConvergeSession {
|
|
92
95
|
/**
|
|
93
|
-
* Before the app-roll: runtime identities + secrets, stack invariants,
|
|
94
|
-
* access. False when a step the install cannot work without failed —
|
|
96
|
+
* Before the app-roll: runtime identities + secrets, stack invariants,
|
|
97
|
+
* public_role's Cloud SQL role memberships (#633), MCP access. False when a step the install cannot work without failed —
|
|
95
98
|
* nothing app-side changed then.
|
|
96
99
|
*/
|
|
97
100
|
beforeAppRoll(): Promise<boolean>;
|
|
@@ -195,6 +198,25 @@ export function startConverge(
|
|
|
195
198
|
return true;
|
|
196
199
|
}
|
|
197
200
|
|
|
201
|
+
async function publicRoleIsolation(): Promise<boolean> {
|
|
202
|
+
// #633 — the Admin API made public_role a member of cloudsqlsuperuser.
|
|
203
|
+
// Revoked here, before migrations, because the migration job's role
|
|
204
|
+
// audit is what verifies it and fails the upgrade while it remains.
|
|
205
|
+
const s = r.step("Removing public_role from the Cloud SQL superuser group...");
|
|
206
|
+
const outcome = await revokePublicRoleMemberships(
|
|
207
|
+
{ projectId: install.projectId, env: install.env },
|
|
208
|
+
runDeps,
|
|
209
|
+
);
|
|
210
|
+
if (outcome.status === "failed") {
|
|
211
|
+
s.failed("public_role's role memberships could not be revoked. Aborting upgrade.");
|
|
212
|
+
r.error(` ${outcome.error ?? ""}`);
|
|
213
|
+
r.warn("No traffic was shifted and no migrations ran. Fix the above and re-run.");
|
|
214
|
+
return false;
|
|
215
|
+
}
|
|
216
|
+
s.done(`public_role holds no database roles (${outcome.detail ?? "revoked"})`);
|
|
217
|
+
return true;
|
|
218
|
+
}
|
|
219
|
+
|
|
198
220
|
async function mcpAccess(): Promise<void> {
|
|
199
221
|
// Issue #37 — the MCP service account + its IAP/token-creator bindings.
|
|
200
222
|
// Also what the deep health check signs its IAP credential as. A failure
|
|
@@ -227,6 +249,29 @@ export function startConverge(
|
|
|
227
249
|
leaveUntouched.push(MCP_ENV_VAR);
|
|
228
250
|
}
|
|
229
251
|
|
|
252
|
+
async function urlMapRedirectsAccess(): Promise<void> {
|
|
253
|
+
// #587 — on `gcp` the admin writes the redirects table into the URL map
|
|
254
|
+
// on Confirm-publish, through a custom role the stack declares. Without
|
|
255
|
+
// it Confirm-publish fails loudly (403); the rest of the install works,
|
|
256
|
+
// so a failure here warns.
|
|
257
|
+
if (install.provider !== "gcp") return;
|
|
258
|
+
const s = r.step("Ensuring the admin may write redirects into the URL map...");
|
|
259
|
+
const access = await ensureUrlMapRedirectsAccess({
|
|
260
|
+
projectId: install.projectId,
|
|
261
|
+
env: install.env,
|
|
262
|
+
...runDeps,
|
|
263
|
+
});
|
|
264
|
+
if (access.ok) {
|
|
265
|
+
s.done(`URL-map redirects role ready (${access.role})`);
|
|
266
|
+
return;
|
|
267
|
+
}
|
|
268
|
+
s.warned(`URL-map redirects role not set up: ${access.error}`);
|
|
269
|
+
r.warn(
|
|
270
|
+
"The upgrade continues; Confirm-publish fails with a permission error until this succeeds. Re-run upgrade with an account that may manage IAM roles.",
|
|
271
|
+
);
|
|
272
|
+
incomplete = true;
|
|
273
|
+
}
|
|
274
|
+
|
|
230
275
|
/** The admin's IAP resource: the service on gcp-firebase, the LB backend on gcp. */
|
|
231
276
|
async function adminIapResource(): Promise<IapResource | null> {
|
|
232
277
|
if (install.provider === "gcp-firebase") {
|
|
@@ -372,7 +417,9 @@ export function startConverge(
|
|
|
372
417
|
if (deps.provisionerIdentity && !(await deps.provisionerIdentity())) return false;
|
|
373
418
|
if (!(await runtimeIdentities())) return false;
|
|
374
419
|
if (!(await stackInvariants())) return false;
|
|
420
|
+
if (!(await publicRoleIsolation())) return false;
|
|
375
421
|
await mcpAccess();
|
|
422
|
+
await urlMapRedirectsAccess();
|
|
376
423
|
return true;
|
|
377
424
|
},
|
|
378
425
|
|
|
@@ -13,6 +13,7 @@
|
|
|
13
13
|
* (epic #696) takes the digests from the signed release manifest instead.
|
|
14
14
|
*/
|
|
15
15
|
|
|
16
|
+
import { IMAGE_DIGEST } from "@caelo-cms/shared";
|
|
16
17
|
import { confirm, isCancel, log, spinner } from "@clack/prompts";
|
|
17
18
|
import { green, red } from "kleur/colors";
|
|
18
19
|
|
|
@@ -62,8 +63,9 @@ export function cosignVerifyArgv(imageRef: string): string[] {
|
|
|
62
63
|
];
|
|
63
64
|
}
|
|
64
65
|
|
|
65
|
-
|
|
66
|
-
|
|
66
|
+
// A `sha256:<64 hex>` image / blob digest — one definition, shared with
|
|
67
|
+
// the release manifest schema.
|
|
68
|
+
export { IMAGE_DIGEST };
|
|
67
69
|
|
|
68
70
|
/** Whether `s` is a well-formed `sha256:<64 hex>` digest. */
|
|
69
71
|
export function isImageDigest(s: string): boolean {
|