@caelo-cms/provisioning 0.10.44 → 0.10.46
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/adc-impersonation.d.ts +28 -0
- package/dist/adc-impersonation.d.ts.map +1 -0
- package/dist/adc-impersonation.js +126 -0
- package/dist/adc-impersonation.js.map +1 -0
- package/dist/checkpoint-secrets.d.ts.map +1 -1
- package/dist/checkpoint-secrets.js +2 -1
- package/dist/checkpoint-secrets.js.map +1 -1
- package/dist/cli.js +199 -33
- package/dist/cli.js.map +1 -1
- package/dist/compose.d.ts +45 -0
- package/dist/compose.d.ts.map +1 -1
- package/dist/compose.js +112 -29
- package/dist/compose.js.map +1 -1
- package/dist/gcloud-retry.d.ts +5 -1
- package/dist/gcloud-retry.d.ts.map +1 -1
- package/dist/gcloud-retry.js.map +1 -1
- package/dist/gcloud.d.ts +2 -1
- package/dist/gcloud.d.ts.map +1 -1
- package/dist/gcloud.js +11 -13
- package/dist/gcloud.js.map +1 -1
- package/dist/index.d.ts +3 -0
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +3 -0
- package/dist/index.js.map +1 -1
- package/dist/infra-contract.d.ts +24 -0
- package/dist/infra-contract.d.ts.map +1 -0
- package/dist/infra-contract.js +25 -0
- package/dist/infra-contract.js.map +1 -0
- package/dist/install-state.d.ts +57 -2
- package/dist/install-state.d.ts.map +1 -1
- package/dist/install-state.js +142 -22
- package/dist/install-state.js.map +1 -1
- package/dist/lifecycle.d.ts +6 -0
- package/dist/lifecycle.d.ts.map +1 -1
- package/dist/lifecycle.js +185 -819
- package/dist/lifecycle.js.map +1 -1
- package/dist/migration-runner.d.ts +3 -0
- package/dist/migration-runner.d.ts.map +1 -1
- package/dist/migration-runner.js +7 -5
- package/dist/migration-runner.js.map +1 -1
- package/dist/ops-backend.d.ts +46 -0
- package/dist/ops-backend.d.ts.map +1 -0
- package/dist/ops-backend.js +370 -0
- package/dist/ops-backend.js.map +1 -0
- package/dist/provisioner-identity.d.ts +103 -0
- package/dist/provisioner-identity.d.ts.map +1 -0
- package/dist/provisioner-identity.js +323 -0
- package/dist/provisioner-identity.js.map +1 -0
- package/dist/regions.d.ts.map +1 -1
- package/dist/regions.js +2 -1
- package/dist/regions.js.map +1 -1
- package/dist/release-contract.d.ts +37 -0
- package/dist/release-contract.d.ts.map +1 -0
- package/dist/release-contract.js +55 -0
- package/dist/release-contract.js.map +1 -0
- package/dist/release-manifest.d.ts +140 -0
- package/dist/release-manifest.d.ts.map +1 -0
- package/dist/release-manifest.js +319 -0
- package/dist/release-manifest.js.map +1 -0
- package/dist/self-command.d.ts +16 -0
- package/dist/self-command.d.ts.map +1 -0
- package/dist/self-command.js +20 -0
- package/dist/self-command.js.map +1 -0
- package/dist/self-hosted-release.d.ts +81 -0
- package/dist/self-hosted-release.d.ts.map +1 -0
- package/dist/self-hosted-release.js +148 -0
- package/dist/self-hosted-release.js.map +1 -0
- package/dist/self-hosted-upgrade.d.ts +91 -0
- package/dist/self-hosted-upgrade.d.ts.map +1 -0
- package/dist/self-hosted-upgrade.js +355 -0
- package/dist/self-hosted-upgrade.js.map +1 -0
- package/dist/stack-converge.d.ts.map +1 -1
- package/dist/stack-converge.js +2 -1
- package/dist/stack-converge.js.map +1 -1
- package/dist/state-backend.d.ts +103 -0
- package/dist/state-backend.d.ts.map +1 -0
- package/dist/state-backend.js +180 -0
- package/dist/state-backend.js.map +1 -0
- package/dist/state-store.d.ts +55 -0
- package/dist/state-store.d.ts.map +1 -0
- package/dist/state-store.js +140 -0
- package/dist/state-store.js.map +1 -0
- package/dist/upgrade/app-roll.d.ts +164 -0
- package/dist/upgrade/app-roll.d.ts.map +1 -0
- package/dist/upgrade/app-roll.js +270 -0
- package/dist/upgrade/app-roll.js.map +1 -0
- package/dist/upgrade/backup.d.ts +48 -0
- package/dist/upgrade/backup.d.ts.map +1 -0
- package/dist/upgrade/backup.js +71 -0
- package/dist/upgrade/backup.js.map +1 -0
- package/dist/upgrade/converge.d.ts +45 -0
- package/dist/upgrade/converge.d.ts.map +1 -0
- package/dist/upgrade/converge.js +287 -0
- package/dist/upgrade/converge.js.map +1 -0
- package/dist/upgrade/gcp-lookup.d.ts +23 -0
- package/dist/upgrade/gcp-lookup.d.ts.map +1 -0
- package/dist/upgrade/gcp-lookup.js +95 -0
- package/dist/upgrade/gcp-lookup.js.map +1 -0
- package/dist/upgrade/health.d.ts +99 -0
- package/dist/upgrade/health.d.ts.map +1 -0
- package/dist/upgrade/health.js +214 -0
- package/dist/upgrade/health.js.map +1 -0
- package/dist/upgrade/release-images.d.ts +125 -0
- package/dist/upgrade/release-images.d.ts.map +1 -0
- package/dist/upgrade/release-images.js +315 -0
- package/dist/upgrade/release-images.js.map +1 -0
- package/dist/upgrade/reporter.d.ts +17 -0
- package/dist/upgrade/reporter.d.ts.map +1 -0
- package/dist/upgrade/reporter.js +28 -0
- package/dist/upgrade/reporter.js.map +1 -0
- package/dist/upgrade/upgrade.d.ts +79 -0
- package/dist/upgrade/upgrade.d.ts.map +1 -0
- package/dist/upgrade/upgrade.js +105 -0
- package/dist/upgrade/upgrade.js.map +1 -0
- package/dist/wizard.d.ts.map +1 -1
- package/dist/wizard.js +7 -3
- package/dist/wizard.js.map +1 -1
- package/dist/wizards/gcp-cost.d.ts.map +1 -1
- package/dist/wizards/gcp-cost.js +7 -1
- package/dist/wizards/gcp-cost.js.map +1 -1
- package/dist/wizards/gcp-pulumi.d.ts +99 -8
- package/dist/wizards/gcp-pulumi.d.ts.map +1 -1
- package/dist/wizards/gcp-pulumi.js +157 -50
- package/dist/wizards/gcp-pulumi.js.map +1 -1
- package/dist/wizards/gcp.d.ts.map +1 -1
- package/dist/wizards/gcp.js +182 -58
- package/dist/wizards/gcp.js.map +1 -1
- package/package.json +4 -3
- package/stacks/self-hosted/README.md +4 -0
- package/stacks/self-hosted/index.ts +10 -2
|
@@ -0,0 +1,148 @@
|
|
|
1
|
+
// SPDX-License-Identifier: MPL-2.0
|
|
2
|
+
/**
|
|
3
|
+
* #696 Phase 4 — which Caelo release a self-hosted install runs, and how a
|
|
4
|
+
* release becomes a pair of verified image digests.
|
|
5
|
+
*
|
|
6
|
+
* `.caelo/release.json` is the install metadata for self-hosted (the cloud
|
|
7
|
+
* providers keep theirs in `~/.caelo-<id>/install.json`): the release
|
|
8
|
+
* version, the digest-pinned admin + gateway image references the Compose
|
|
9
|
+
* file runs, and a short history of installs/upgrades/rollbacks. It holds no
|
|
10
|
+
* secrets (config.json does), so it is safe to read from tooling and
|
|
11
|
+
* backups; writes refuse secret material like install.json's do.
|
|
12
|
+
*
|
|
13
|
+
* Every image reference that reaches the Compose file comes through
|
|
14
|
+
* {@link resolveVerifiedRelease}: the tag is resolved to a digest in GHCR
|
|
15
|
+
* (anonymously) and that digest — not the tag — is cosign-verified against
|
|
16
|
+
* the Caelo release workflow identity. `init` and `upgrade` share it.
|
|
17
|
+
*/
|
|
18
|
+
import { existsSync, readFileSync } from "node:fs";
|
|
19
|
+
import { join } from "node:path";
|
|
20
|
+
import { assertNoSecrets, writeJsonAtomic } from "./checkpoint-secrets.js";
|
|
21
|
+
import { ghcrImageDigest, ghcrImageRef } from "./compose.js";
|
|
22
|
+
import { selfCommand } from "./self-command.js";
|
|
23
|
+
import { cosignVerifyArgv, GHCR_NAMESPACE, GHCR_REGISTRY, RELEASE_SERVICES, resolveRegistryDigest, } from "./upgrade/release-images.js";
|
|
24
|
+
/** The real runner: Bun.spawn; a missing binary is exit code 127, not a throw. */
|
|
25
|
+
export const bunHostRunner = {
|
|
26
|
+
async run(argv, opts) {
|
|
27
|
+
try {
|
|
28
|
+
const proc = Bun.spawn([...argv], {
|
|
29
|
+
stdout: opts?.stdoutPath ? Bun.file(opts.stdoutPath) : "pipe",
|
|
30
|
+
stderr: "pipe",
|
|
31
|
+
});
|
|
32
|
+
const stdout = opts?.stdoutPath
|
|
33
|
+
? ""
|
|
34
|
+
: await new Response(proc.stdout).text();
|
|
35
|
+
const stderr = await new Response(proc.stderr).text();
|
|
36
|
+
return { exitCode: await proc.exited, stdout, stderr };
|
|
37
|
+
}
|
|
38
|
+
catch (e) {
|
|
39
|
+
return { exitCode: 127, stdout: "", stderr: e instanceof Error ? e.message : String(e) };
|
|
40
|
+
}
|
|
41
|
+
},
|
|
42
|
+
};
|
|
43
|
+
const HISTORY_LIMIT = 20;
|
|
44
|
+
export function releasePath(caeloDir) {
|
|
45
|
+
return join(caeloDir, "release.json");
|
|
46
|
+
}
|
|
47
|
+
/** The images of a recorded release, as the Compose generator takes them. */
|
|
48
|
+
export function releaseImages(release) {
|
|
49
|
+
return { mode: "release", admin: release.images.admin, gateway: release.images.gateway };
|
|
50
|
+
}
|
|
51
|
+
/**
|
|
52
|
+
* The install's recorded release, or null when none is recorded (a dev-mode
|
|
53
|
+
* install, or one created before #696 that still runs the repo bind mount).
|
|
54
|
+
* Throws on a malformed record rather than running a guessed image.
|
|
55
|
+
*/
|
|
56
|
+
export function readRelease(caeloDir) {
|
|
57
|
+
const path = releasePath(caeloDir);
|
|
58
|
+
if (!existsSync(path))
|
|
59
|
+
return null;
|
|
60
|
+
const raw = JSON.parse(readFileSync(path, "utf8"));
|
|
61
|
+
const admin = raw.images?.admin ?? "";
|
|
62
|
+
const gateway = raw.images?.gateway ?? "";
|
|
63
|
+
if (typeof raw.version !== "string" ||
|
|
64
|
+
!ghcrImageDigest("admin", admin) ||
|
|
65
|
+
!ghcrImageDigest("gateway", gateway)) {
|
|
66
|
+
throw new Error(`${path} is malformed (needs version + digest-pinned ghcr.io/caelo-cms images); fix it or re-run \`${selfCommand("upgrade", "--version", "<x.y.z>")}\`.`);
|
|
67
|
+
}
|
|
68
|
+
return {
|
|
69
|
+
version: raw.version,
|
|
70
|
+
images: { admin, gateway },
|
|
71
|
+
installedAt: raw.installedAt ?? "",
|
|
72
|
+
history: Array.isArray(raw.history) ? raw.history : [],
|
|
73
|
+
};
|
|
74
|
+
}
|
|
75
|
+
/**
|
|
76
|
+
* Record that the install now runs `images` of `version` (or, for a
|
|
77
|
+
* rollback, keeps running the previous ones), appending to the history.
|
|
78
|
+
*/
|
|
79
|
+
export function recordRelease(caeloDir, next, entry, now = new Date()) {
|
|
80
|
+
const prior = readRelease(caeloDir);
|
|
81
|
+
const at = now.toISOString();
|
|
82
|
+
const record = {
|
|
83
|
+
version: next.version,
|
|
84
|
+
images: { admin: next.images.admin, gateway: next.images.gateway },
|
|
85
|
+
installedAt: prior && prior.version === next.version && entry.outcome === "rolled-back"
|
|
86
|
+
? prior.installedAt
|
|
87
|
+
: at,
|
|
88
|
+
history: [...(prior?.history ?? []), { at, ...entry }].slice(-HISTORY_LIMIT),
|
|
89
|
+
};
|
|
90
|
+
assertNoSecrets(record, "release.json");
|
|
91
|
+
writeJsonAtomic(releasePath(caeloDir), record);
|
|
92
|
+
return record;
|
|
93
|
+
}
|
|
94
|
+
/** Record a failed-and-rolled-back upgrade without changing the running release. */
|
|
95
|
+
export function recordRollback(caeloDir, current, attempted, detail, now = new Date()) {
|
|
96
|
+
recordRelease(caeloDir, { version: current.version, images: releaseImages(current) }, { from: current.version, to: attempted, outcome: "rolled-back", detail }, now);
|
|
97
|
+
}
|
|
98
|
+
/**
|
|
99
|
+
* Resolve the admin + gateway images of release `tag` in GHCR to digests,
|
|
100
|
+
* then cosign-verify each digest against the Caelo release workflow
|
|
101
|
+
* identity (unless `skipVerify`, the escape hatch for forks). The returned
|
|
102
|
+
* references are digest-pinned, so what was verified is exactly what runs.
|
|
103
|
+
*/
|
|
104
|
+
export async function resolveVerifiedRelease(tag, deps) {
|
|
105
|
+
const refs = {};
|
|
106
|
+
for (const service of RELEASE_SERVICES) {
|
|
107
|
+
deps.onProgress?.(`Resolving ${GHCR_REGISTRY}/${GHCR_NAMESPACE}/${service}:${tag}`);
|
|
108
|
+
const r = await resolveRegistryDigest(GHCR_REGISTRY, `${GHCR_NAMESPACE}/${service}`, tag, deps.fetchImpl);
|
|
109
|
+
if (!r.ok) {
|
|
110
|
+
return {
|
|
111
|
+
ok: false,
|
|
112
|
+
error: `Could not resolve ${GHCR_REGISTRY}/${GHCR_NAMESPACE}/${service}:${tag} (${r.reason}).\n` +
|
|
113
|
+
"Check the version exists at https://github.com/caelo-cms/caelo-cms/releases and pass it as --version x.y.z.",
|
|
114
|
+
};
|
|
115
|
+
}
|
|
116
|
+
refs[service] = ghcrImageRef(service, r.digest);
|
|
117
|
+
}
|
|
118
|
+
const images = {
|
|
119
|
+
mode: "release",
|
|
120
|
+
admin: refs.admin ?? "",
|
|
121
|
+
gateway: refs.gateway ?? "",
|
|
122
|
+
};
|
|
123
|
+
if (deps.skipVerify)
|
|
124
|
+
return { ok: true, images };
|
|
125
|
+
const probe = await deps.runner.run(["cosign", "version"]);
|
|
126
|
+
if (probe.exitCode !== 0) {
|
|
127
|
+
return {
|
|
128
|
+
ok: false,
|
|
129
|
+
error: "cosign is required to verify the Caelo release image signatures and was not found on PATH.\n" +
|
|
130
|
+
"Install it (https://docs.sigstore.dev/cosign/system_config/installation/ — e.g. `brew install cosign`,\n" +
|
|
131
|
+
"or the release binary for Linux), then re-run. --skip-verify skips the check (NOT for production).",
|
|
132
|
+
};
|
|
133
|
+
}
|
|
134
|
+
for (const service of RELEASE_SERVICES) {
|
|
135
|
+
deps.onProgress?.(`Verifying the signature of ${service}`);
|
|
136
|
+
const v = await deps.runner.run(cosignVerifyArgv(images[service]));
|
|
137
|
+
if (v.exitCode !== 0) {
|
|
138
|
+
return {
|
|
139
|
+
ok: false,
|
|
140
|
+
error: `cosign verify FAILED for ${images[service]}: its signature does not match the Caelo release workflow.\n` +
|
|
141
|
+
"Either the registry was compromised, the image is a fork's, or sigstore had an outage (retry).\n" +
|
|
142
|
+
`cosign: ${v.stderr.trim().slice(0, 500)}`,
|
|
143
|
+
};
|
|
144
|
+
}
|
|
145
|
+
}
|
|
146
|
+
return { ok: true, images };
|
|
147
|
+
}
|
|
148
|
+
//# sourceMappingURL=self-hosted-release.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"self-hosted-release.js","sourceRoot":"","sources":["../src/self-hosted-release.ts"],"names":[],"mappings":"AAAA,mCAAmC;AAEnC;;;;;;;;;;;;;;;GAeG;AAEH,OAAO,EAAE,UAAU,EAAE,YAAY,EAAE,MAAM,SAAS,CAAC;AACnD,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AACjC,OAAO,EAAE,eAAe,EAAE,eAAe,EAAE,MAAM,yBAAyB,CAAC;AAC3E,OAAO,EAAE,eAAe,EAAE,YAAY,EAAsB,MAAM,cAAc,CAAC;AACjF,OAAO,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAChD,OAAO,EACL,gBAAgB,EAChB,cAAc,EACd,aAAa,EACb,gBAAgB,EAChB,qBAAqB,GACtB,MAAM,6BAA6B,CAAC;AAqBrC,kFAAkF;AAClF,MAAM,CAAC,MAAM,aAAa,GAAe;IACvC,KAAK,CAAC,GAAG,CAAC,IAAI,EAAE,IAAI;QAClB,IAAI,CAAC;YACH,MAAM,IAAI,GAAG,GAAG,CAAC,KAAK,CAAC,CAAC,GAAG,IAAI,CAAC,EAAE;gBAChC,MAAM,EAAE,IAAI,EAAE,UAAU,CAAC,CAAC,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,MAAM;gBAC7D,MAAM,EAAE,MAAM;aACf,CAAC,CAAC;YACH,MAAM,MAAM,GAAG,IAAI,EAAE,UAAU;gBAC7B,CAAC,CAAC,EAAE;gBACJ,CAAC,CAAC,MAAM,IAAI,QAAQ,CAAC,IAAI,CAAC,MAAwB,CAAC,CAAC,IAAI,EAAE,CAAC;YAC7D,MAAM,MAAM,GAAG,MAAM,IAAI,QAAQ,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,CAAC;YACtD,OAAO,EAAE,QAAQ,EAAE,MAAM,IAAI,CAAC,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,CAAC;QACzD,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACX,OAAO,EAAE,QAAQ,EAAE,GAAG,EAAE,MAAM,EAAE,EAAE,EAAE,MAAM,EAAE,CAAC,YAAY,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC;QAC3F,CAAC;IACH,CAAC;CACF,CAAC;AAqBF,MAAM,aAAa,GAAG,EAAE,CAAC;AAEzB,MAAM,UAAU,WAAW,CAAC,QAAgB;IAC1C,OAAO,IAAI,CAAC,QAAQ,EAAE,cAAc,CAAC,CAAC;AACxC,CAAC;AAED,6EAA6E;AAC7E,MAAM,UAAU,aAAa,CAAC,OAA0B;IACtD,OAAO,EAAE,IAAI,EAAE,SAAS,EAAE,KAAK,EAAE,OAAO,CAAC,MAAM,CAAC,KAAK,EAAE,OAAO,EAAE,OAAO,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC;AAC3F,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,WAAW,CAAC,QAAgB;IAC1C,MAAM,IAAI,GAAG,WAAW,CAAC,QAAQ,CAAC,CAAC;IACnC,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC;QAAE,OAAO,IAAI,CAAC;IACnC,MAAM,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC,YAAY,CAAC,IAAI,EAAE,MAAM,CAAC,CAA+B,CAAC;IACjF,MAAM,KAAK,GAAG,GAAG,CAAC,MAAM,EAAE,KAAK,IAAI,EAAE,CAAC;IACtC,MAAM,OAAO,GAAG,GAAG,CAAC,MAAM,EAAE,OAAO,IAAI,EAAE,CAAC;IAC1C,IACE,OAAO,GAAG,CAAC,OAAO,KAAK,QAAQ;QAC/B,CAAC,eAAe,CAAC,OAAO,EAAE,KAAK,CAAC;QAChC,CAAC,eAAe,CAAC,SAAS,EAAE,OAAO,CAAC,EACpC,CAAC;QACD,MAAM,IAAI,KAAK,CACb,GAAG,IAAI,8FAA8F,WAAW,CAAC,SAAS,EAAE,WAAW,EAAE,SAAS,CAAC,KAAK,CACzJ,CAAC;IACJ,CAAC;IACD,OAAO;QACL,OAAO,EAAE,GAAG,CAAC,OAAO;QACpB,MAAM,EAAE,EAAE,KAAK,EAAE,OAAO,EAAE;QAC1B,WAAW,EAAE,GAAG,CAAC,WAAW,IAAI,EAAE;QAClC,OAAO,EAAE,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE;KACvD,CAAC;AACJ,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,aAAa,CAC3B,QAAgB,EAChB,IAAgD,EAChD,KAAsC,EACtC,MAAY,IAAI,IAAI,EAAE;IAEtB,MAAM,KAAK,GAAG,WAAW,CAAC,QAAQ,CAAC,CAAC;IACpC,MAAM,EAAE,GAAG,GAAG,CAAC,WAAW,EAAE,CAAC;IAC7B,MAAM,MAAM,GAAsB;QAChC,OAAO,EAAE,IAAI,CAAC,OAAO;QACrB,MAAM,EAAE,EAAE,KAAK,EAAE,IAAI,CAAC,MAAM,CAAC,KAAK,EAAE,OAAO,EAAE,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE;QAClE,WAAW,EACT,KAAK,IAAI,KAAK,CAAC,OAAO,KAAK,IAAI,CAAC,OAAO,IAAI,KAAK,CAAC,OAAO,KAAK,aAAa;YACxE,CAAC,CAAC,KAAK,CAAC,WAAW;YACnB,CAAC,CAAC,EAAE;QACR,OAAO,EAAE,CAAC,GAAG,CAAC,KAAK,EAAE,OAAO,IAAI,EAAE,CAAC,EAAE,EAAE,EAAE,EAAE,GAAG,KAAK,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,aAAa,CAAC;KAC7E,CAAC;IACF,eAAe,CAAC,MAAM,EAAE,cAAc,CAAC,CAAC;IACxC,eAAe,CAAC,WAAW,CAAC,QAAQ,CAAC,EAAE,MAAM,CAAC,CAAC;IAC/C,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,oFAAoF;AACpF,MAAM,UAAU,cAAc,CAC5B,QAAgB,EAChB,OAA0B,EAC1B,SAAiB,EACjB,MAAc,EACd,MAAY,IAAI,IAAI,EAAE;IAEtB,aAAa,CACX,QAAQ,EACR,EAAE,OAAO,EAAE,OAAO,CAAC,OAAO,EAAE,MAAM,EAAE,aAAa,CAAC,OAAO,CAAC,EAAE,EAC5D,EAAE,IAAI,EAAE,OAAO,CAAC,OAAO,EAAE,EAAE,EAAE,SAAS,EAAE,OAAO,EAAE,aAAa,EAAE,MAAM,EAAE,EACxE,GAAG,CACJ,CAAC;AACJ,CAAC;AAMD;;;;;GAKG;AACH,MAAM,CAAC,KAAK,UAAU,sBAAsB,CAC1C,GAAW,EACX,IAKC;IAED,MAAM,IAAI,GAAiD,EAAE,CAAC;IAC9D,KAAK,MAAM,OAAO,IAAI,gBAAgB,EAAE,CAAC;QACvC,IAAI,CAAC,UAAU,EAAE,CAAC,aAAa,aAAa,IAAI,cAAc,IAAI,OAAO,IAAI,GAAG,EAAE,CAAC,CAAC;QACpF,MAAM,CAAC,GAAG,MAAM,qBAAqB,CACnC,aAAa,EACb,GAAG,cAAc,IAAI,OAAO,EAAE,EAC9B,GAAG,EACH,IAAI,CAAC,SAAS,CACf,CAAC;QACF,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC;YACV,OAAO;gBACL,EAAE,EAAE,KAAK;gBACT,KAAK,EACH,qBAAqB,aAAa,IAAI,cAAc,IAAI,OAAO,IAAI,GAAG,KAAK,CAAC,CAAC,MAAM,MAAM;oBACzF,6GAA6G;aAChH,CAAC;QACJ,CAAC;QACD,IAAI,CAAC,OAAO,CAAC,GAAG,YAAY,CAAC,OAAO,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC;IAClD,CAAC;IACD,MAAM,MAAM,GAAkB;QAC5B,IAAI,EAAE,SAAS;QACf,KAAK,EAAE,IAAI,CAAC,KAAK,IAAI,EAAE;QACvB,OAAO,EAAE,IAAI,CAAC,OAAO,IAAI,EAAE;KAC5B,CAAC;IACF,IAAI,IAAI,CAAC,UAAU;QAAE,OAAO,EAAE,EAAE,EAAE,IAAI,EAAE,MAAM,EAAE,CAAC;IAEjD,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,QAAQ,EAAE,SAAS,CAAC,CAAC,CAAC;IAC3D,IAAI,KAAK,CAAC,QAAQ,KAAK,CAAC,EAAE,CAAC;QACzB,OAAO;YACL,EAAE,EAAE,KAAK;YACT,KAAK,EACH,8FAA8F;gBAC9F,0GAA0G;gBAC1G,oGAAoG;SACvG,CAAC;IACJ,CAAC;IACD,KAAK,MAAM,OAAO,IAAI,gBAAgB,EAAE,CAAC;QACvC,IAAI,CAAC,UAAU,EAAE,CAAC,8BAA8B,OAAO,EAAE,CAAC,CAAC;QAC3D,MAAM,CAAC,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,gBAAgB,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;QACnE,IAAI,CAAC,CAAC,QAAQ,KAAK,CAAC,EAAE,CAAC;YACrB,OAAO;gBACL,EAAE,EAAE,KAAK;gBACT,KAAK,EACH,4BAA4B,MAAM,CAAC,OAAO,CAAC,8DAA8D;oBACzG,kGAAkG;oBAClG,WAAW,CAAC,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE;aAC7C,CAAC;QACJ,CAAC;IACH,CAAC;IACD,OAAO,EAAE,EAAE,EAAE,IAAI,EAAE,MAAM,EAAE,CAAC;AAC9B,CAAC"}
|
|
@@ -0,0 +1,91 @@
|
|
|
1
|
+
import { type ReleaseImages } from "./compose.js";
|
|
2
|
+
import { type HostRunner } from "./self-hosted-release.js";
|
|
3
|
+
/** Progress output; the CLI wires it to the terminal, tests to an array. */
|
|
4
|
+
export interface UpgradeLog {
|
|
5
|
+
step(message: string): void;
|
|
6
|
+
info(message: string): void;
|
|
7
|
+
warn(message: string): void;
|
|
8
|
+
error(message: string): void;
|
|
9
|
+
}
|
|
10
|
+
export interface SelfHostedUpgradeDeps {
|
|
11
|
+
/** The install's `.caelo/` directory (docker-compose.yml, release.json, backups/). */
|
|
12
|
+
readonly caeloDir: string;
|
|
13
|
+
readonly runner: HostRunner;
|
|
14
|
+
/** Renders the Compose file for `images` from the install's config (secrets included). */
|
|
15
|
+
readonly renderCompose: (images: ReleaseImages) => string;
|
|
16
|
+
readonly log: UpgradeLog;
|
|
17
|
+
readonly fetchImpl?: typeof fetch;
|
|
18
|
+
readonly sleep?: (ms: number) => Promise<void>;
|
|
19
|
+
readonly now?: () => Date;
|
|
20
|
+
/** Health probe attempts per service and the pause between them. */
|
|
21
|
+
readonly healthAttempts?: number;
|
|
22
|
+
readonly healthIntervalMs?: number;
|
|
23
|
+
}
|
|
24
|
+
export interface SelfHostedUpgradeOpts {
|
|
25
|
+
readonly version?: string;
|
|
26
|
+
readonly channel?: "stable" | "rc" | "beta";
|
|
27
|
+
readonly skipVerify?: boolean;
|
|
28
|
+
}
|
|
29
|
+
export type SelfHostedUpgradeResult = {
|
|
30
|
+
readonly ok: true;
|
|
31
|
+
readonly status: "upgraded" | "already-current";
|
|
32
|
+
readonly version: string;
|
|
33
|
+
} | {
|
|
34
|
+
readonly ok: false;
|
|
35
|
+
readonly stage: UpgradeStage;
|
|
36
|
+
readonly error: string;
|
|
37
|
+
/** Whether the previous release is running again (true when nothing had changed yet). */
|
|
38
|
+
readonly previousRunning: boolean;
|
|
39
|
+
};
|
|
40
|
+
type UpgradeStage = "target" | "verify" | "converge" | "backup" | "pull" | "migrate" | "roll-admin" | "roll-gateway";
|
|
41
|
+
/** Where an install from before #696 keeps its original Compose file during the move. */
|
|
42
|
+
export declare function legacyComposePath(caeloDir: string): string;
|
|
43
|
+
/** Where the media of an install from before #696 lives: the checkout's admin cwd. */
|
|
44
|
+
export declare function legacyMediaDir(caeloDir: string): string;
|
|
45
|
+
/**
|
|
46
|
+
* Copy (never move) a legacy install's uploads from the checkout
|
|
47
|
+
* (`apps/admin/data/media`, the admin's cwd under the repo bind mount) into
|
|
48
|
+
* `.caelo/media/`, which the release-mode admin mounts as MEDIA_ROOT_DIR.
|
|
49
|
+
* Skipped when there is nothing to copy or `.caelo/media/` already has
|
|
50
|
+
* content (a previous run copied it).
|
|
51
|
+
*/
|
|
52
|
+
export declare function adoptLegacyMedia(caeloDir: string): {
|
|
53
|
+
ok: true;
|
|
54
|
+
copied: string | null;
|
|
55
|
+
} | {
|
|
56
|
+
ok: false;
|
|
57
|
+
error: string;
|
|
58
|
+
};
|
|
59
|
+
export declare function composePath(caeloDir: string): string;
|
|
60
|
+
/**
|
|
61
|
+
* Upgrade a self-hosted install to `opts.version` (or the newest stable
|
|
62
|
+
* release). Never throws for an expected failure; the result says which
|
|
63
|
+
* stage failed and whether the previous release is serving again.
|
|
64
|
+
*/
|
|
65
|
+
export declare function upgradeSelfHosted(opts: SelfHostedUpgradeOpts, deps: SelfHostedUpgradeDeps): Promise<SelfHostedUpgradeResult>;
|
|
66
|
+
/**
|
|
67
|
+
* pg_dump both databases (custom format) plus the cluster's roles into
|
|
68
|
+
* `.caelo/backups/pre-upgrade-<timestamp>/` (mode 700). Every dump must exit
|
|
69
|
+
* 0 and be non-empty, or the upgrade does not start.
|
|
70
|
+
*/
|
|
71
|
+
export declare function backupDatabases(caeloDir: string, runner: HostRunner, at: Date): Promise<{
|
|
72
|
+
ok: true;
|
|
73
|
+
dir: string;
|
|
74
|
+
} | {
|
|
75
|
+
ok: false;
|
|
76
|
+
error: string;
|
|
77
|
+
}>;
|
|
78
|
+
/**
|
|
79
|
+
* Apply the committed migrations (cms_admin, then cms_public) with the
|
|
80
|
+
* admin image the Compose file names, as a one-off container — the
|
|
81
|
+
* self-hosted equivalent of the GCP `caelo-migrate-<target>` Cloud Run job.
|
|
82
|
+
* drizzle's bookkeeping makes re-runs apply only new migrations.
|
|
83
|
+
*/
|
|
84
|
+
export declare function runMigrations(caeloDir: string, runner: HostRunner): Promise<{
|
|
85
|
+
ok: true;
|
|
86
|
+
} | {
|
|
87
|
+
ok: false;
|
|
88
|
+
error: string;
|
|
89
|
+
}>;
|
|
90
|
+
export {};
|
|
91
|
+
//# sourceMappingURL=self-hosted-upgrade.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"self-hosted-upgrade.d.ts","sourceRoot":"","sources":["../src/self-hosted-upgrade.ts"],"names":[],"mappings":"AA2CA,OAAO,EAAa,KAAK,aAAa,EAAE,MAAM,cAAc,CAAC;AAE7D,OAAO,EACL,KAAK,UAAU,EAMhB,MAAM,0BAA0B,CAAC;AAMlC,4EAA4E;AAC5E,MAAM,WAAW,UAAU;IACzB,IAAI,CAAC,OAAO,EAAE,MAAM,GAAG,IAAI,CAAC;IAC5B,IAAI,CAAC,OAAO,EAAE,MAAM,GAAG,IAAI,CAAC;IAC5B,IAAI,CAAC,OAAO,EAAE,MAAM,GAAG,IAAI,CAAC;IAC5B,KAAK,CAAC,OAAO,EAAE,MAAM,GAAG,IAAI,CAAC;CAC9B;AAED,MAAM,WAAW,qBAAqB;IACpC,sFAAsF;IACtF,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;IAC1B,QAAQ,CAAC,MAAM,EAAE,UAAU,CAAC;IAC5B,0FAA0F;IAC1F,QAAQ,CAAC,aAAa,EAAE,CAAC,MAAM,EAAE,aAAa,KAAK,MAAM,CAAC;IAC1D,QAAQ,CAAC,GAAG,EAAE,UAAU,CAAC;IACzB,QAAQ,CAAC,SAAS,CAAC,EAAE,OAAO,KAAK,CAAC;IAClC,QAAQ,CAAC,KAAK,CAAC,EAAE,CAAC,EAAE,EAAE,MAAM,KAAK,OAAO,CAAC,IAAI,CAAC,CAAC;IAC/C,QAAQ,CAAC,GAAG,CAAC,EAAE,MAAM,IAAI,CAAC;IAC1B,oEAAoE;IACpE,QAAQ,CAAC,cAAc,CAAC,EAAE,MAAM,CAAC;IACjC,QAAQ,CAAC,gBAAgB,CAAC,EAAE,MAAM,CAAC;CACpC;AAED,MAAM,WAAW,qBAAqB;IACpC,QAAQ,CAAC,OAAO,CAAC,EAAE,MAAM,CAAC;IAC1B,QAAQ,CAAC,OAAO,CAAC,EAAE,QAAQ,GAAG,IAAI,GAAG,MAAM,CAAC;IAC5C,QAAQ,CAAC,UAAU,CAAC,EAAE,OAAO,CAAC;CAC/B;AAED,MAAM,MAAM,uBAAuB,GAC/B;IAAE,QAAQ,CAAC,EAAE,EAAE,IAAI,CAAC;IAAC,QAAQ,CAAC,MAAM,EAAE,UAAU,GAAG,iBAAiB,CAAC;IAAC,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAA;CAAE,GAChG;IACE,QAAQ,CAAC,EAAE,EAAE,KAAK,CAAC;IACnB,QAAQ,CAAC,KAAK,EAAE,YAAY,CAAC;IAC7B,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,yFAAyF;IACzF,QAAQ,CAAC,eAAe,EAAE,OAAO,CAAC;CACnC,CAAC;AAEN,KAAK,YAAY,GACb,QAAQ,GACR,QAAQ,GACR,UAAU,GACV,QAAQ,GACR,MAAM,GACN,SAAS,GACT,YAAY,GACZ,cAAc,CAAC;AAEnB,yFAAyF;AACzF,wBAAgB,iBAAiB,CAAC,QAAQ,EAAE,MAAM,GAAG,MAAM,CAE1D;AAoBD,sFAAsF;AACtF,wBAAgB,cAAc,CAAC,QAAQ,EAAE,MAAM,GAAG,MAAM,CAEvD;AAED;;;;;;GAMG;AACH,wBAAgB,gBAAgB,CAC9B,QAAQ,EAAE,MAAM,GACf;IAAE,EAAE,EAAE,IAAI,CAAC;IAAC,MAAM,EAAE,MAAM,GAAG,IAAI,CAAA;CAAE,GAAG;IAAE,EAAE,EAAE,KAAK,CAAC;IAAC,KAAK,EAAE,MAAM,CAAA;CAAE,CAcpE;AAqBD,wBAAgB,WAAW,CAAC,QAAQ,EAAE,MAAM,GAAG,MAAM,CAEpD;AAOD;;;;GAIG;AACH,wBAAsB,iBAAiB,CACrC,IAAI,EAAE,qBAAqB,EAC3B,IAAI,EAAE,qBAAqB,GAC1B,OAAO,CAAC,uBAAuB,CAAC,CAuIlC;AA8BD;;;;GAIG;AACH,wBAAsB,eAAe,CACnC,QAAQ,EAAE,MAAM,EAChB,MAAM,EAAE,UAAU,EAClB,EAAE,EAAE,IAAI,GACP,OAAO,CAAC;IAAE,EAAE,EAAE,IAAI,CAAC;IAAC,GAAG,EAAE,MAAM,CAAA;CAAE,GAAG;IAAE,EAAE,EAAE,KAAK,CAAC;IAAC,KAAK,EAAE,MAAM,CAAA;CAAE,CAAC,CAkCnE;AAED;;;;;GAKG;AACH,wBAAsB,aAAa,CACjC,QAAQ,EAAE,MAAM,EAChB,MAAM,EAAE,UAAU,GACjB,OAAO,CAAC;IAAE,EAAE,EAAE,IAAI,CAAA;CAAE,GAAG;IAAE,EAAE,EAAE,KAAK,CAAC;IAAC,KAAK,EAAE,MAAM,CAAA;CAAE,CAAC,CAqBtD"}
|
|
@@ -0,0 +1,355 @@
|
|
|
1
|
+
// SPDX-License-Identifier: MPL-2.0
|
|
2
|
+
/**
|
|
3
|
+
* #696 Phase 4 — `cms-provision upgrade` for a self-hosted Compose install.
|
|
4
|
+
*
|
|
5
|
+
* Order, and why:
|
|
6
|
+
* 1. resolve the target release to digests + cosign-verify them — nothing
|
|
7
|
+
* on the host changes until the images are known-good;
|
|
8
|
+
* 2. back up both databases (pg_dump, hard-fail) — the last moment the
|
|
9
|
+
* schema is guaranteed to match the running images;
|
|
10
|
+
* 3. write the new digests into the Compose file, `docker compose pull`;
|
|
11
|
+
* 4. run the migrations from the NEW admin image (`docker compose run`,
|
|
12
|
+
* the same `migrate.ts` entrypoint the GCP migrate job runs) while the
|
|
13
|
+
* old containers keep serving — migrations are expand-only, so the
|
|
14
|
+
* running release works on the new schema;
|
|
15
|
+
* 5. recreate the admin, health-check it (`/_caelo/health`, which proves
|
|
16
|
+
* the DB pool and roles); then the gateway (`/healthz`);
|
|
17
|
+
* 6. record the release in `.caelo/release.json`.
|
|
18
|
+
* Any failure after step 3 puts the previous Compose file back; once a
|
|
19
|
+
* container was recreated the previous images are started again and
|
|
20
|
+
* health-checked (rollback). The database is never restored automatically:
|
|
21
|
+
* the expand-only migrations keep the previous release working, and the
|
|
22
|
+
* backup path is printed for the case they do not.
|
|
23
|
+
*
|
|
24
|
+
* Idempotent: a target equal to the recorded release is a no-op; a run that
|
|
25
|
+
* stopped half-way is simply repeated (pull, migrate and `up` converge).
|
|
26
|
+
*
|
|
27
|
+
* pgBackRest is NOT used: the Compose file's sidecar has no access to the
|
|
28
|
+
* Postgres data volume and no stanza is ever created, so `pg_dump` (from the
|
|
29
|
+
* Postgres container itself) is the backup that actually works here.
|
|
30
|
+
*/
|
|
31
|
+
import { chmodSync, cpSync, existsSync, mkdirSync, readdirSync, readFileSync, statSync, writeFileSync, } from "node:fs";
|
|
32
|
+
import { join } from "node:path";
|
|
33
|
+
import { MEDIA_DIR } from "./compose.js";
|
|
34
|
+
import { selfCommand } from "./self-command.js";
|
|
35
|
+
import { readRelease, recordRelease, recordRollback, releaseImages, resolveVerifiedRelease, } from "./self-hosted-release.js";
|
|
36
|
+
import { fetchLatestReleaseTag, RELEASE_SERVICES, releaseTag } from "./upgrade/release-images.js";
|
|
37
|
+
/** The Compose service names of the two Caelo services. */
|
|
38
|
+
const COMPOSE_SERVICE = { admin: "caelo-admin", gateway: "caelo-gateway" };
|
|
39
|
+
/** Where an install from before #696 keeps its original Compose file during the move. */
|
|
40
|
+
export function legacyComposePath(caeloDir) {
|
|
41
|
+
return join(caeloDir, "docker-compose.pre-release.yml");
|
|
42
|
+
}
|
|
43
|
+
/**
|
|
44
|
+
* The legacy (repo-mounted) Compose file, saved to
|
|
45
|
+
* {@link legacyComposePath} on the first upgrade and read from there on
|
|
46
|
+
* every retry — after a stopped run the live file may already name the
|
|
47
|
+
* release images, which would make a rollback a no-op.
|
|
48
|
+
*/
|
|
49
|
+
function legacyComposeCheckpoint(caeloDir, live) {
|
|
50
|
+
const saved = legacyComposePath(caeloDir);
|
|
51
|
+
try {
|
|
52
|
+
// `wx`: create only — an existing checkpoint (an earlier run's) wins.
|
|
53
|
+
writeFileSync(saved, live, { mode: 0o600, flag: "wx" });
|
|
54
|
+
return live;
|
|
55
|
+
}
|
|
56
|
+
catch (e) {
|
|
57
|
+
if (e.code !== "EEXIST")
|
|
58
|
+
throw e;
|
|
59
|
+
return readFileSync(saved, "utf8");
|
|
60
|
+
}
|
|
61
|
+
}
|
|
62
|
+
/** Where the media of an install from before #696 lives: the checkout's admin cwd. */
|
|
63
|
+
export function legacyMediaDir(caeloDir) {
|
|
64
|
+
return join(caeloDir, "..", "apps", "admin", "data", "media");
|
|
65
|
+
}
|
|
66
|
+
/**
|
|
67
|
+
* Copy (never move) a legacy install's uploads from the checkout
|
|
68
|
+
* (`apps/admin/data/media`, the admin's cwd under the repo bind mount) into
|
|
69
|
+
* `.caelo/media/`, which the release-mode admin mounts as MEDIA_ROOT_DIR.
|
|
70
|
+
* Skipped when there is nothing to copy or `.caelo/media/` already has
|
|
71
|
+
* content (a previous run copied it).
|
|
72
|
+
*/
|
|
73
|
+
export function adoptLegacyMedia(caeloDir) {
|
|
74
|
+
const from = legacyMediaDir(caeloDir);
|
|
75
|
+
const to = join(caeloDir, MEDIA_DIR);
|
|
76
|
+
if (!existsSync(from))
|
|
77
|
+
return { ok: true, copied: null };
|
|
78
|
+
if (existsSync(to) && readdirSync(to).length > 0)
|
|
79
|
+
return { ok: true, copied: null };
|
|
80
|
+
try {
|
|
81
|
+
cpSync(from, to, { recursive: true, preserveTimestamps: true });
|
|
82
|
+
return { ok: true, copied: to };
|
|
83
|
+
}
|
|
84
|
+
catch (e) {
|
|
85
|
+
return {
|
|
86
|
+
ok: false,
|
|
87
|
+
error: `Could not copy the uploaded media from ${from} to ${to} (${e instanceof Error ? e.message : String(e)}). The upgrade did not start.`,
|
|
88
|
+
};
|
|
89
|
+
}
|
|
90
|
+
}
|
|
91
|
+
/** The file's size in bytes, 0 when it does not exist. */
|
|
92
|
+
function fileSize(path) {
|
|
93
|
+
try {
|
|
94
|
+
return statSync(path).size;
|
|
95
|
+
}
|
|
96
|
+
catch {
|
|
97
|
+
return 0;
|
|
98
|
+
}
|
|
99
|
+
}
|
|
100
|
+
/** The file's content, or null when it does not exist. */
|
|
101
|
+
function readIfExists(path) {
|
|
102
|
+
try {
|
|
103
|
+
return readFileSync(path, "utf8");
|
|
104
|
+
}
|
|
105
|
+
catch (e) {
|
|
106
|
+
if (e.code === "ENOENT")
|
|
107
|
+
return null;
|
|
108
|
+
throw e;
|
|
109
|
+
}
|
|
110
|
+
}
|
|
111
|
+
export function composePath(caeloDir) {
|
|
112
|
+
return join(caeloDir, "docker-compose.yml");
|
|
113
|
+
}
|
|
114
|
+
/** `docker compose -f <file>` — every Compose call targets the install's file explicitly. */
|
|
115
|
+
function compose(caeloDir, ...args) {
|
|
116
|
+
return ["docker", "compose", "-f", composePath(caeloDir), ...args];
|
|
117
|
+
}
|
|
118
|
+
/**
|
|
119
|
+
* Upgrade a self-hosted install to `opts.version` (or the newest stable
|
|
120
|
+
* release). Never throws for an expected failure; the result says which
|
|
121
|
+
* stage failed and whether the previous release is serving again.
|
|
122
|
+
*/
|
|
123
|
+
export async function upgradeSelfHosted(opts, deps) {
|
|
124
|
+
const { caeloDir, runner, log } = deps;
|
|
125
|
+
const now = deps.now ?? (() => new Date());
|
|
126
|
+
const current = readRelease(caeloDir);
|
|
127
|
+
const file = composePath(caeloDir);
|
|
128
|
+
const live = readIfExists(file);
|
|
129
|
+
if (live === null) {
|
|
130
|
+
return fail("target", `${file} does not exist — run \`${selfCommand("init")}\` first.`, true);
|
|
131
|
+
}
|
|
132
|
+
// The file to restore on failure: the recorded release re-rendered (so a
|
|
133
|
+
// run that stopped after writing new digests cannot "restore" those), or —
|
|
134
|
+
// for an install from before #696 with no record — the legacy file,
|
|
135
|
+
// checkpointed once so a retry after a stopped run still has it.
|
|
136
|
+
const previousCompose = current
|
|
137
|
+
? deps.renderCompose(releaseImages(current))
|
|
138
|
+
: legacyComposeCheckpoint(caeloDir, live);
|
|
139
|
+
// 1. Target release → verified digests.
|
|
140
|
+
const target = await targetVersion(opts, deps.fetchImpl);
|
|
141
|
+
if (!target.ok)
|
|
142
|
+
return fail("target", target.error, true);
|
|
143
|
+
log.step(`Upgrading ${current ? `${current.version} → ` : "(repo-mounted install) → "}${target.version}`);
|
|
144
|
+
const resolved = await resolveVerifiedRelease(target.tag, {
|
|
145
|
+
runner,
|
|
146
|
+
...(opts.skipVerify ? { skipVerify: true } : {}),
|
|
147
|
+
...(deps.fetchImpl ? { fetchImpl: deps.fetchImpl } : {}),
|
|
148
|
+
onProgress: (m) => log.info(m),
|
|
149
|
+
});
|
|
150
|
+
if (!resolved.ok)
|
|
151
|
+
return fail("verify", resolved.error, true);
|
|
152
|
+
if (opts.skipVerify)
|
|
153
|
+
log.warn("--skip-verify set — image signatures NOT verified.");
|
|
154
|
+
const images = resolved.images;
|
|
155
|
+
if (current &&
|
|
156
|
+
current.images.admin === images.admin &&
|
|
157
|
+
current.images.gateway === images.gateway &&
|
|
158
|
+
live === previousCompose) {
|
|
159
|
+
// Recorded, and the Compose file agrees. `up` converges containers a
|
|
160
|
+
// stopped run or a failed rollback may have left on other images (a
|
|
161
|
+
// no-op when they already match); the health checks prove it serves.
|
|
162
|
+
log.info(`Already on ${current.version}; checking the running containers match and are healthy.`);
|
|
163
|
+
const converged = await startBoth(deps);
|
|
164
|
+
if (!converged.ok) {
|
|
165
|
+
return fail("converge", `${converged.error}\nThe recorded release ${current.version} is not serving healthily — check \`docker compose logs\`.`, false);
|
|
166
|
+
}
|
|
167
|
+
log.info(`Already on ${current.version} (${shortDigest(images.admin)}); nothing to do.`);
|
|
168
|
+
return { ok: true, status: "already-current", version: current.version };
|
|
169
|
+
}
|
|
170
|
+
// Otherwise (a different target, or a Compose file a stopped run left
|
|
171
|
+
// with other digests) the full flow below converges the install.
|
|
172
|
+
// Rendered before the backup so a config problem stops the run before
|
|
173
|
+
// anything on the host is touched.
|
|
174
|
+
const nextCompose = deps.renderCompose(images);
|
|
175
|
+
// 2. Backup — before anything on the host changes.
|
|
176
|
+
log.step("Backing up the databases (pg_dump)");
|
|
177
|
+
const backup = await backupDatabases(caeloDir, runner, now());
|
|
178
|
+
if (!backup.ok)
|
|
179
|
+
return fail("backup", backup.error, true);
|
|
180
|
+
log.info(`Backup written to ${backup.dir}`);
|
|
181
|
+
if (!current) {
|
|
182
|
+
const media = adoptLegacyMedia(caeloDir);
|
|
183
|
+
if (!media.ok)
|
|
184
|
+
return fail("backup", media.error, true);
|
|
185
|
+
if (media.copied)
|
|
186
|
+
log.info(`Copied the uploaded media into ${media.copied}`);
|
|
187
|
+
}
|
|
188
|
+
// 3. New digests into the Compose file; pull them.
|
|
189
|
+
writeFileSync(file, nextCompose, { mode: 0o600 });
|
|
190
|
+
log.step("Pulling the new images");
|
|
191
|
+
const pull = await runner.run(compose(caeloDir, "pull", COMPOSE_SERVICE.admin, COMPOSE_SERVICE.gateway));
|
|
192
|
+
if (pull.exitCode !== 0) {
|
|
193
|
+
restoreCompose(file, previousCompose);
|
|
194
|
+
return fail("pull", `docker compose pull failed: ${tail(pull.stderr)}`, true);
|
|
195
|
+
}
|
|
196
|
+
// 4. Migrations from the new admin image; the old containers keep serving.
|
|
197
|
+
log.step("Running database migrations with the new admin image");
|
|
198
|
+
const migrated = await runMigrations(caeloDir, runner);
|
|
199
|
+
if (!migrated.ok) {
|
|
200
|
+
restoreCompose(file, previousCompose);
|
|
201
|
+
return fail("migrate", `${migrated.error}\nThe previous release keeps running. Database backup: ${backup.dir}`, true);
|
|
202
|
+
}
|
|
203
|
+
// 5. Roll admin, then gateway, each health-checked.
|
|
204
|
+
for (const service of RELEASE_SERVICES) {
|
|
205
|
+
const stage = service === "admin" ? "roll-admin" : "roll-gateway";
|
|
206
|
+
log.step(`Restarting ${service} on ${shortDigest(images[service])}`);
|
|
207
|
+
const up = await runner.run(compose(caeloDir, "up", "-d", "--no-deps", COMPOSE_SERVICE[service]));
|
|
208
|
+
const healthy = up.exitCode === 0
|
|
209
|
+
? await waitHealthy(service, caeloDir, deps)
|
|
210
|
+
: { ok: false, error: `docker compose up failed: ${tail(up.stderr)}` };
|
|
211
|
+
if (!healthy.ok) {
|
|
212
|
+
log.error(`${service} is not healthy on ${target.version}: ${healthy.error}`);
|
|
213
|
+
const restored = await rollback(file, previousCompose, deps);
|
|
214
|
+
if (current) {
|
|
215
|
+
recordRollback(caeloDir, current, target.version, `${stage}: ${healthy.error}`, now());
|
|
216
|
+
}
|
|
217
|
+
return fail(stage, `${healthy.error}\n${restored
|
|
218
|
+
? "Rolled back: the previous release is running and healthy again."
|
|
219
|
+
: "ROLLBACK FAILED — the previous images did not come back healthy. Check `docker compose ps` / `docker compose logs`."}\nDatabase backup: ${backup.dir}`, restored);
|
|
220
|
+
}
|
|
221
|
+
log.info(`${service} healthy ✓`);
|
|
222
|
+
}
|
|
223
|
+
// 6. Record.
|
|
224
|
+
recordRelease(caeloDir, { version: target.version, images }, { from: current?.version ?? null, to: target.version, outcome: "upgraded" }, now());
|
|
225
|
+
log.step(`Upgraded to ${target.version}. Backup kept at ${backup.dir}`);
|
|
226
|
+
return { ok: true, status: "upgraded", version: target.version };
|
|
227
|
+
}
|
|
228
|
+
function fail(stage, error, previousRunning) {
|
|
229
|
+
return { ok: false, stage, error, previousRunning };
|
|
230
|
+
}
|
|
231
|
+
/** The tag to resolve and the version to record: explicit, channel, or newest stable. */
|
|
232
|
+
async function targetVersion(opts, fetchImpl) {
|
|
233
|
+
if (opts.version || opts.channel === "rc" || opts.channel === "beta") {
|
|
234
|
+
const tag = releaseTag(opts);
|
|
235
|
+
return { ok: true, tag, version: tag };
|
|
236
|
+
}
|
|
237
|
+
const latest = await fetchLatestReleaseTag(fetchImpl);
|
|
238
|
+
if (!latest.ok) {
|
|
239
|
+
return {
|
|
240
|
+
ok: false,
|
|
241
|
+
error: `Could not look up the newest Caelo release (${latest.reason}). Pass --version x.y.z.`,
|
|
242
|
+
};
|
|
243
|
+
}
|
|
244
|
+
const tag = releaseTag({ version: latest.tag });
|
|
245
|
+
return { ok: true, tag, version: tag };
|
|
246
|
+
}
|
|
247
|
+
/**
|
|
248
|
+
* pg_dump both databases (custom format) plus the cluster's roles into
|
|
249
|
+
* `.caelo/backups/pre-upgrade-<timestamp>/` (mode 700). Every dump must exit
|
|
250
|
+
* 0 and be non-empty, or the upgrade does not start.
|
|
251
|
+
*/
|
|
252
|
+
export async function backupDatabases(caeloDir, runner, at) {
|
|
253
|
+
const ready = await runner.run(compose(caeloDir, "exec", "-T", "postgres", "pg_isready", "-U", "caelo"));
|
|
254
|
+
if (ready.exitCode !== 0) {
|
|
255
|
+
return {
|
|
256
|
+
ok: false,
|
|
257
|
+
error: "Postgres is not running, so no backup can be taken and the upgrade will not start.\n" +
|
|
258
|
+
`Start it with \`docker compose -f ${composePath(caeloDir)} up -d postgres\` and re-run.`,
|
|
259
|
+
};
|
|
260
|
+
}
|
|
261
|
+
const dir = join(caeloDir, "backups", `pre-upgrade-${at.toISOString().replace(/[:.]/g, "-")}`);
|
|
262
|
+
mkdirSync(dir, { recursive: true, mode: 0o700 });
|
|
263
|
+
chmodSync(dir, 0o700);
|
|
264
|
+
const dumps = [
|
|
265
|
+
["globals.sql", ["pg_dumpall", "-U", "caelo", "--globals-only"]],
|
|
266
|
+
["cms_admin.dump", ["pg_dump", "-U", "caelo", "-Fc", "cms_admin"]],
|
|
267
|
+
["cms_public.dump", ["pg_dump", "-U", "caelo", "-Fc", "cms_public"]],
|
|
268
|
+
];
|
|
269
|
+
for (const [name, cmd] of dumps) {
|
|
270
|
+
const out = join(dir, name);
|
|
271
|
+
const r = await runner.run(compose(caeloDir, "exec", "-T", "postgres", ...cmd), {
|
|
272
|
+
stdoutPath: out,
|
|
273
|
+
});
|
|
274
|
+
if (r.exitCode !== 0 || fileSize(out) === 0) {
|
|
275
|
+
return {
|
|
276
|
+
ok: false,
|
|
277
|
+
error: `Backup failed (${cmd.join(" ")} → ${out}): ${tail(r.stderr) || "empty dump"}. The upgrade did not start.`,
|
|
278
|
+
};
|
|
279
|
+
}
|
|
280
|
+
chmodSync(out, 0o600);
|
|
281
|
+
}
|
|
282
|
+
return { ok: true, dir };
|
|
283
|
+
}
|
|
284
|
+
/**
|
|
285
|
+
* Apply the committed migrations (cms_admin, then cms_public) with the
|
|
286
|
+
* admin image the Compose file names, as a one-off container — the
|
|
287
|
+
* self-hosted equivalent of the GCP `caelo-migrate-<target>` Cloud Run job.
|
|
288
|
+
* drizzle's bookkeeping makes re-runs apply only new migrations.
|
|
289
|
+
*/
|
|
290
|
+
export async function runMigrations(caeloDir, runner) {
|
|
291
|
+
for (const target of ["admin", "public"]) {
|
|
292
|
+
const r = await runner.run(compose(caeloDir, "run", "--rm", "--no-deps", "-T", COMPOSE_SERVICE.admin, "bun", "--bun", "/app/packages/migrations/src/migrate.ts", target));
|
|
293
|
+
if (r.exitCode !== 0) {
|
|
294
|
+
return { ok: false, error: `${target} migrations failed: ${tail(r.stderr || r.stdout)}` };
|
|
295
|
+
}
|
|
296
|
+
}
|
|
297
|
+
return { ok: true };
|
|
298
|
+
}
|
|
299
|
+
/**
|
|
300
|
+
* In-container probe scripts (`bun -e`): the containers publish no host
|
|
301
|
+
* ports, so the probe runs where the service listens. The admin's
|
|
302
|
+
* `/_caelo/health` checks the DB pool + roles; the gateway's `/healthz` is
|
|
303
|
+
* its liveness endpoint.
|
|
304
|
+
*/
|
|
305
|
+
const HEALTH_PROBE = {
|
|
306
|
+
admin: 'const r=await fetch("http://127.0.0.1:"+(process.env.PORT||"3000")+"/_caelo/health").catch(()=>null);process.exit(r&&r.ok?0:1)',
|
|
307
|
+
gateway: 'const r=await fetch("http://127.0.0.1:"+(process.env.PORT||process.env.GATEWAY_PORT||"8090")+"/healthz").catch(()=>null);process.exit(r&&r.ok?0:1)',
|
|
308
|
+
};
|
|
309
|
+
async function waitHealthy(service, caeloDir, deps) {
|
|
310
|
+
const attempts = deps.healthAttempts ?? 45;
|
|
311
|
+
const interval = deps.healthIntervalMs ?? 2000;
|
|
312
|
+
const sleep = deps.sleep ?? ((ms) => new Promise((r) => setTimeout(r, ms)));
|
|
313
|
+
let last = "";
|
|
314
|
+
for (let i = 0; i < attempts; i++) {
|
|
315
|
+
const r = await deps.runner.run(compose(caeloDir, "exec", "-T", COMPOSE_SERVICE[service], "bun", "-e", HEALTH_PROBE[service]));
|
|
316
|
+
if (r.exitCode === 0)
|
|
317
|
+
return { ok: true };
|
|
318
|
+
last = tail(r.stderr);
|
|
319
|
+
if (i < attempts - 1)
|
|
320
|
+
await sleep(interval);
|
|
321
|
+
}
|
|
322
|
+
return {
|
|
323
|
+
ok: false,
|
|
324
|
+
error: `health check did not pass after ${attempts} attempts${last ? ` (${last})` : ""}`,
|
|
325
|
+
};
|
|
326
|
+
}
|
|
327
|
+
/** Put the previous Compose file back, restart both services on it, health-check them. */
|
|
328
|
+
async function rollback(file, previousCompose, deps) {
|
|
329
|
+
deps.log.warn("Rolling back to the previous images");
|
|
330
|
+
restoreCompose(file, previousCompose);
|
|
331
|
+
return (await startBoth(deps)).ok;
|
|
332
|
+
}
|
|
333
|
+
/** `up -d` both services on the current Compose file, then health-check each. */
|
|
334
|
+
async function startBoth(deps) {
|
|
335
|
+
const up = await deps.runner.run(compose(deps.caeloDir, "up", "-d", "--no-deps", COMPOSE_SERVICE.admin, COMPOSE_SERVICE.gateway));
|
|
336
|
+
if (up.exitCode !== 0)
|
|
337
|
+
return { ok: false, error: `docker compose up failed: ${tail(up.stderr)}` };
|
|
338
|
+
for (const service of RELEASE_SERVICES) {
|
|
339
|
+
const healthy = await waitHealthy(service, deps.caeloDir, deps);
|
|
340
|
+
if (!healthy.ok)
|
|
341
|
+
return { ok: false, error: `${service}: ${healthy.error}` };
|
|
342
|
+
}
|
|
343
|
+
return { ok: true };
|
|
344
|
+
}
|
|
345
|
+
function restoreCompose(file, previous) {
|
|
346
|
+
writeFileSync(file, previous, { mode: 0o600 });
|
|
347
|
+
}
|
|
348
|
+
function shortDigest(ref) {
|
|
349
|
+
const at = ref.indexOf("@sha256:");
|
|
350
|
+
return at === -1 ? ref : ref.slice(at + 1, at + 20);
|
|
351
|
+
}
|
|
352
|
+
function tail(s) {
|
|
353
|
+
return s.trim().split("\n").slice(-5).join("\n").slice(0, 800);
|
|
354
|
+
}
|
|
355
|
+
//# sourceMappingURL=self-hosted-upgrade.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"self-hosted-upgrade.js","sourceRoot":"","sources":["../src/self-hosted-upgrade.ts"],"names":[],"mappings":"AAAA,mCAAmC;AAEnC;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA4BG;AAEH,OAAO,EACL,SAAS,EACT,MAAM,EACN,UAAU,EACV,SAAS,EACT,WAAW,EACX,YAAY,EACZ,QAAQ,EACR,aAAa,GACd,MAAM,SAAS,CAAC;AACjB,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AACjC,OAAO,EAAE,SAAS,EAAsB,MAAM,cAAc,CAAC;AAC7D,OAAO,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAChD,OAAO,EAEL,WAAW,EACX,aAAa,EACb,cAAc,EACd,aAAa,EACb,sBAAsB,GACvB,MAAM,0BAA0B,CAAC;AAClC,OAAO,EAAE,qBAAqB,EAAE,gBAAgB,EAAE,UAAU,EAAE,MAAM,6BAA6B,CAAC;AAElG,2DAA2D;AAC3D,MAAM,eAAe,GAAG,EAAE,KAAK,EAAE,aAAa,EAAE,OAAO,EAAE,eAAe,EAAW,CAAC;AAmDpF,yFAAyF;AACzF,MAAM,UAAU,iBAAiB,CAAC,QAAgB;IAChD,OAAO,IAAI,CAAC,QAAQ,EAAE,gCAAgC,CAAC,CAAC;AAC1D,CAAC;AAED;;;;;GAKG;AACH,SAAS,uBAAuB,CAAC,QAAgB,EAAE,IAAY;IAC7D,MAAM,KAAK,GAAG,iBAAiB,CAAC,QAAQ,CAAC,CAAC;IAC1C,IAAI,CAAC;QACH,sEAAsE;QACtE,aAAa,CAAC,KAAK,EAAE,IAAI,EAAE,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC;QACxD,OAAO,IAAI,CAAC;IACd,CAAC;IAAC,OAAO,CAAC,EAAE,CAAC;QACX,IAAK,CAA2B,CAAC,IAAI,KAAK,QAAQ;YAAE,MAAM,CAAC,CAAC;QAC5D,OAAO,YAAY,CAAC,KAAK,EAAE,MAAM,CAAC,CAAC;IACrC,CAAC;AACH,CAAC;AAED,sFAAsF;AACtF,MAAM,UAAU,cAAc,CAAC,QAAgB;IAC7C,OAAO,IAAI,CAAC,QAAQ,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,OAAO,CAAC,CAAC;AAChE,CAAC;AAED;;;;;;GAMG;AACH,MAAM,UAAU,gBAAgB,CAC9B,QAAgB;IAEhB,MAAM,IAAI,GAAG,cAAc,CAAC,QAAQ,CAAC,CAAC;IACtC,MAAM,EAAE,GAAG,IAAI,CAAC,QAAQ,EAAE,SAAS,CAAC,CAAC;IACrC,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC;QAAE,OAAO,EAAE,EAAE,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC;IACzD,IAAI,UAAU,CAAC,EAAE,CAAC,IAAI,WAAW,CAAC,EAAE,CAAC,CAAC,MAAM,GAAG,CAAC;QAAE,OAAO,EAAE,EAAE,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC;IACpF,IAAI,CAAC;QACH,MAAM,CAAC,IAAI,EAAE,EAAE,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,kBAAkB,EAAE,IAAI,EAAE,CAAC,CAAC;QAChE,OAAO,EAAE,EAAE,EAAE,IAAI,EAAE,MAAM,EAAE,EAAE,EAAE,CAAC;IAClC,CAAC;IAAC,OAAO,CAAC,EAAE,CAAC;QACX,OAAO;YACL,EAAE,EAAE,KAAK;YACT,KAAK,EAAE,0CAA0C,IAAI,OAAO,EAAE,KAAK,CAAC,YAAY,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,+BAA+B;SAC7I,CAAC;IACJ,CAAC;AACH,CAAC;AAED,0DAA0D;AAC1D,SAAS,QAAQ,CAAC,IAAY;IAC5B,IAAI,CAAC;QACH,OAAO,QAAQ,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC;IAC7B,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,CAAC,CAAC;IACX,CAAC;AACH,CAAC;AAED,0DAA0D;AAC1D,SAAS,YAAY,CAAC,IAAY;IAChC,IAAI,CAAC;QACH,OAAO,YAAY,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;IACpC,CAAC;IAAC,OAAO,CAAC,EAAE,CAAC;QACX,IAAK,CAA2B,CAAC,IAAI,KAAK,QAAQ;YAAE,OAAO,IAAI,CAAC;QAChE,MAAM,CAAC,CAAC;IACV,CAAC;AACH,CAAC;AAED,MAAM,UAAU,WAAW,CAAC,QAAgB;IAC1C,OAAO,IAAI,CAAC,QAAQ,EAAE,oBAAoB,CAAC,CAAC;AAC9C,CAAC;AAED,6FAA6F;AAC7F,SAAS,OAAO,CAAC,QAAgB,EAAE,GAAG,IAAc;IAClD,OAAO,CAAC,QAAQ,EAAE,SAAS,EAAE,IAAI,EAAE,WAAW,CAAC,QAAQ,CAAC,EAAE,GAAG,IAAI,CAAC,CAAC;AACrE,CAAC;AAED;;;;GAIG;AACH,MAAM,CAAC,KAAK,UAAU,iBAAiB,CACrC,IAA2B,EAC3B,IAA2B;IAE3B,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE,GAAG,EAAE,GAAG,IAAI,CAAC;IACvC,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC,IAAI,IAAI,EAAE,CAAC,CAAC;IAC3C,MAAM,OAAO,GAAG,WAAW,CAAC,QAAQ,CAAC,CAAC;IACtC,MAAM,IAAI,GAAG,WAAW,CAAC,QAAQ,CAAC,CAAC;IACnC,MAAM,IAAI,GAAG,YAAY,CAAC,IAAI,CAAC,CAAC;IAChC,IAAI,IAAI,KAAK,IAAI,EAAE,CAAC;QAClB,OAAO,IAAI,CAAC,QAAQ,EAAE,GAAG,IAAI,2BAA2B,WAAW,CAAC,MAAM,CAAC,WAAW,EAAE,IAAI,CAAC,CAAC;IAChG,CAAC;IACD,yEAAyE;IACzE,2EAA2E;IAC3E,oEAAoE;IACpE,iEAAiE;IACjE,MAAM,eAAe,GAAG,OAAO;QAC7B,CAAC,CAAC,IAAI,CAAC,aAAa,CAAC,aAAa,CAAC,OAAO,CAAC,CAAC;QAC5C,CAAC,CAAC,uBAAuB,CAAC,QAAQ,EAAE,IAAI,CAAC,CAAC;IAE5C,wCAAwC;IACxC,MAAM,MAAM,GAAG,MAAM,aAAa,CAAC,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,CAAC;IACzD,IAAI,CAAC,MAAM,CAAC,EAAE;QAAE,OAAO,IAAI,CAAC,QAAQ,EAAE,MAAM,CAAC,KAAK,EAAE,IAAI,CAAC,CAAC;IAC1D,GAAG,CAAC,IAAI,CACN,aAAa,OAAO,CAAC,CAAC,CAAC,GAAG,OAAO,CAAC,OAAO,KAAK,CAAC,CAAC,CAAC,2BAA2B,GAAG,MAAM,CAAC,OAAO,EAAE,CAChG,CAAC;IACF,MAAM,QAAQ,GAAG,MAAM,sBAAsB,CAAC,MAAM,CAAC,GAAG,EAAE;QACxD,MAAM;QACN,GAAG,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC,CAAC,EAAE,UAAU,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QAChD,GAAG,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE,SAAS,EAAE,IAAI,CAAC,SAAS,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QACxD,UAAU,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC;KAC/B,CAAC,CAAC;IACH,IAAI,CAAC,QAAQ,CAAC,EAAE;QAAE,OAAO,IAAI,CAAC,QAAQ,EAAE,QAAQ,CAAC,KAAK,EAAE,IAAI,CAAC,CAAC;IAC9D,IAAI,IAAI,CAAC,UAAU;QAAE,GAAG,CAAC,IAAI,CAAC,oDAAoD,CAAC,CAAC;IACpF,MAAM,MAAM,GAAG,QAAQ,CAAC,MAAM,CAAC;IAC/B,IACE,OAAO;QACP,OAAO,CAAC,MAAM,CAAC,KAAK,KAAK,MAAM,CAAC,KAAK;QACrC,OAAO,CAAC,MAAM,CAAC,OAAO,KAAK,MAAM,CAAC,OAAO;QACzC,IAAI,KAAK,eAAe,EACxB,CAAC;QACD,qEAAqE;QACrE,oEAAoE;QACpE,qEAAqE;QACrE,GAAG,CAAC,IAAI,CACN,cAAc,OAAO,CAAC,OAAO,0DAA0D,CACxF,CAAC;QACF,MAAM,SAAS,GAAG,MAAM,SAAS,CAAC,IAAI,CAAC,CAAC;QACxC,IAAI,CAAC,SAAS,CAAC,EAAE,EAAE,CAAC;YAClB,OAAO,IAAI,CACT,UAAU,EACV,GAAG,SAAS,CAAC,KAAK,0BAA0B,OAAO,CAAC,OAAO,4DAA4D,EACvH,KAAK,CACN,CAAC;QACJ,CAAC;QACD,GAAG,CAAC,IAAI,CAAC,cAAc,OAAO,CAAC,OAAO,KAAK,WAAW,CAAC,MAAM,CAAC,KAAK,CAAC,mBAAmB,CAAC,CAAC;QACzF,OAAO,EAAE,EAAE,EAAE,IAAI,EAAE,MAAM,EAAE,iBAAiB,EAAE,OAAO,EAAE,OAAO,CAAC,OAAO,EAAE,CAAC;IAC3E,CAAC;IACD,sEAAsE;IACtE,iEAAiE;IAEjE,sEAAsE;IACtE,mCAAmC;IACnC,MAAM,WAAW,GAAG,IAAI,CAAC,aAAa,CAAC,MAAM,CAAC,CAAC;IAE/C,mDAAmD;IACnD,GAAG,CAAC,IAAI,CAAC,oCAAoC,CAAC,CAAC;IAC/C,MAAM,MAAM,GAAG,MAAM,eAAe,CAAC,QAAQ,EAAE,MAAM,EAAE,GAAG,EAAE,CAAC,CAAC;IAC9D,IAAI,CAAC,MAAM,CAAC,EAAE;QAAE,OAAO,IAAI,CAAC,QAAQ,EAAE,MAAM,CAAC,KAAK,EAAE,IAAI,CAAC,CAAC;IAC1D,GAAG,CAAC,IAAI,CAAC,qBAAqB,MAAM,CAAC,GAAG,EAAE,CAAC,CAAC;IAC5C,IAAI,CAAC,OAAO,EAAE,CAAC;QACb,MAAM,KAAK,GAAG,gBAAgB,CAAC,QAAQ,CAAC,CAAC;QACzC,IAAI,CAAC,KAAK,CAAC,EAAE;YAAE,OAAO,IAAI,CAAC,QAAQ,EAAE,KAAK,CAAC,KAAK,EAAE,IAAI,CAAC,CAAC;QACxD,IAAI,KAAK,CAAC,MAAM;YAAE,GAAG,CAAC,IAAI,CAAC,kCAAkC,KAAK,CAAC,MAAM,EAAE,CAAC,CAAC;IAC/E,CAAC;IAED,mDAAmD;IACnD,aAAa,CAAC,IAAI,EAAE,WAAW,EAAE,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,CAAC;IAClD,GAAG,CAAC,IAAI,CAAC,wBAAwB,CAAC,CAAC;IACnC,MAAM,IAAI,GAAG,MAAM,MAAM,CAAC,GAAG,CAC3B,OAAO,CAAC,QAAQ,EAAE,MAAM,EAAE,eAAe,CAAC,KAAK,EAAE,eAAe,CAAC,OAAO,CAAC,CAC1E,CAAC;IACF,IAAI,IAAI,CAAC,QAAQ,KAAK,CAAC,EAAE,CAAC;QACxB,cAAc,CAAC,IAAI,EAAE,eAAe,CAAC,CAAC;QACtC,OAAO,IAAI,CAAC,MAAM,EAAE,+BAA+B,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,EAAE,EAAE,IAAI,CAAC,CAAC;IAChF,CAAC;IAED,2EAA2E;IAC3E,GAAG,CAAC,IAAI,CAAC,sDAAsD,CAAC,CAAC;IACjE,MAAM,QAAQ,GAAG,MAAM,aAAa,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC;IACvD,IAAI,CAAC,QAAQ,CAAC,EAAE,EAAE,CAAC;QACjB,cAAc,CAAC,IAAI,EAAE,eAAe,CAAC,CAAC;QACtC,OAAO,IAAI,CACT,SAAS,EACT,GAAG,QAAQ,CAAC,KAAK,0DAA0D,MAAM,CAAC,GAAG,EAAE,EACvF,IAAI,CACL,CAAC;IACJ,CAAC;IAED,oDAAoD;IACpD,KAAK,MAAM,OAAO,IAAI,gBAAgB,EAAE,CAAC;QACvC,MAAM,KAAK,GAAG,OAAO,KAAK,OAAO,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,cAAc,CAAC;QAClE,GAAG,CAAC,IAAI,CAAC,cAAc,OAAO,OAAO,WAAW,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,EAAE,CAAC,CAAC;QACrE,MAAM,EAAE,GAAG,MAAM,MAAM,CAAC,GAAG,CACzB,OAAO,CAAC,QAAQ,EAAE,IAAI,EAAE,IAAI,EAAE,WAAW,EAAE,eAAe,CAAC,OAAO,CAAC,CAAC,CACrE,CAAC;QACF,MAAM,OAAO,GACX,EAAE,CAAC,QAAQ,KAAK,CAAC;YACf,CAAC,CAAC,MAAM,WAAW,CAAC,OAAO,EAAE,QAAQ,EAAE,IAAI,CAAC;YAC5C,CAAC,CAAC,EAAE,EAAE,EAAE,KAAc,EAAE,KAAK,EAAE,6BAA6B,IAAI,CAAC,EAAE,CAAC,MAAM,CAAC,EAAE,EAAE,CAAC;QACpF,IAAI,CAAC,OAAO,CAAC,EAAE,EAAE,CAAC;YAChB,GAAG,CAAC,KAAK,CAAC,GAAG,OAAO,sBAAsB,MAAM,CAAC,OAAO,KAAK,OAAO,CAAC,KAAK,EAAE,CAAC,CAAC;YAC9E,MAAM,QAAQ,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,eAAe,EAAE,IAAI,CAAC,CAAC;YAC7D,IAAI,OAAO,EAAE,CAAC;gBACZ,cAAc,CAAC,QAAQ,EAAE,OAAO,EAAE,MAAM,CAAC,OAAO,EAAE,GAAG,KAAK,KAAK,OAAO,CAAC,KAAK,EAAE,EAAE,GAAG,EAAE,CAAC,CAAC;YACzF,CAAC;YACD,OAAO,IAAI,CACT,KAAK,EACL,GAAG,OAAO,CAAC,KAAK,KACd,QAAQ;gBACN,CAAC,CAAC,iEAAiE;gBACnE,CAAC,CAAC,qHACN,sBAAsB,MAAM,CAAC,GAAG,EAAE,EAClC,QAAQ,CACT,CAAC;QACJ,CAAC;QACD,GAAG,CAAC,IAAI,CAAC,GAAG,OAAO,YAAY,CAAC,CAAC;IACnC,CAAC;IAED,aAAa;IACb,aAAa,CACX,QAAQ,EACR,EAAE,OAAO,EAAE,MAAM,CAAC,OAAO,EAAE,MAAM,EAAE,EACnC,EAAE,IAAI,EAAE,OAAO,EAAE,OAAO,IAAI,IAAI,EAAE,EAAE,EAAE,MAAM,CAAC,OAAO,EAAE,OAAO,EAAE,UAAU,EAAE,EAC3E,GAAG,EAAE,CACN,CAAC;IACF,GAAG,CAAC,IAAI,CAAC,eAAe,MAAM,CAAC,OAAO,oBAAoB,MAAM,CAAC,GAAG,EAAE,CAAC,CAAC;IACxE,OAAO,EAAE,EAAE,EAAE,IAAI,EAAE,MAAM,EAAE,UAAU,EAAE,OAAO,EAAE,MAAM,CAAC,OAAO,EAAE,CAAC;AACnE,CAAC;AAED,SAAS,IAAI,CACX,KAAmB,EACnB,KAAa,EACb,eAAwB;IAExB,OAAO,EAAE,EAAE,EAAE,KAAK,EAAE,KAAK,EAAE,KAAK,EAAE,eAAe,EAAE,CAAC;AACtD,CAAC;AAED,yFAAyF;AACzF,KAAK,UAAU,aAAa,CAC1B,IAA2B,EAC3B,SAAwB;IAExB,IAAI,IAAI,CAAC,OAAO,IAAI,IAAI,CAAC,OAAO,KAAK,IAAI,IAAI,IAAI,CAAC,OAAO,KAAK,MAAM,EAAE,CAAC;QACrE,MAAM,GAAG,GAAG,UAAU,CAAC,IAAI,CAAC,CAAC;QAC7B,OAAO,EAAE,EAAE,EAAE,IAAI,EAAE,GAAG,EAAE,OAAO,EAAE,GAAG,EAAE,CAAC;IACzC,CAAC;IACD,MAAM,MAAM,GAAG,MAAM,qBAAqB,CAAC,SAAS,CAAC,CAAC;IACtD,IAAI,CAAC,MAAM,CAAC,EAAE,EAAE,CAAC;QACf,OAAO;YACL,EAAE,EAAE,KAAK;YACT,KAAK,EAAE,+CAA+C,MAAM,CAAC,MAAM,0BAA0B;SAC9F,CAAC;IACJ,CAAC;IACD,MAAM,GAAG,GAAG,UAAU,CAAC,EAAE,OAAO,EAAE,MAAM,CAAC,GAAG,EAAE,CAAC,CAAC;IAChD,OAAO,EAAE,EAAE,EAAE,IAAI,EAAE,GAAG,EAAE,OAAO,EAAE,GAAG,EAAE,CAAC;AACzC,CAAC;AAED;;;;GAIG;AACH,MAAM,CAAC,KAAK,UAAU,eAAe,CACnC,QAAgB,EAChB,MAAkB,EAClB,EAAQ;IAER,MAAM,KAAK,GAAG,MAAM,MAAM,CAAC,GAAG,CAC5B,OAAO,CAAC,QAAQ,EAAE,MAAM,EAAE,IAAI,EAAE,UAAU,EAAE,YAAY,EAAE,IAAI,EAAE,OAAO,CAAC,CACzE,CAAC;IACF,IAAI,KAAK,CAAC,QAAQ,KAAK,CAAC,EAAE,CAAC;QACzB,OAAO;YACL,EAAE,EAAE,KAAK;YACT,KAAK,EACH,sFAAsF;gBACtF,qCAAqC,WAAW,CAAC,QAAQ,CAAC,+BAA+B;SAC5F,CAAC;IACJ,CAAC;IACD,MAAM,GAAG,GAAG,IAAI,CAAC,QAAQ,EAAE,SAAS,EAAE,eAAe,EAAE,CAAC,WAAW,EAAE,CAAC,OAAO,CAAC,OAAO,EAAE,GAAG,CAAC,EAAE,CAAC,CAAC;IAC/F,SAAS,CAAC,GAAG,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,CAAC;IACjD,SAAS,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC;IACtB,MAAM,KAAK,GAAyB;QAClC,CAAC,aAAa,EAAE,CAAC,YAAY,EAAE,IAAI,EAAE,OAAO,EAAE,gBAAgB,CAAC,CAAC;QAChE,CAAC,gBAAgB,EAAE,CAAC,SAAS,EAAE,IAAI,EAAE,OAAO,EAAE,KAAK,EAAE,WAAW,CAAC,CAAC;QAClE,CAAC,iBAAiB,EAAE,CAAC,SAAS,EAAE,IAAI,EAAE,OAAO,EAAE,KAAK,EAAE,YAAY,CAAC,CAAC;KACrE,CAAC;IACF,KAAK,MAAM,CAAC,IAAI,EAAE,GAAG,CAAC,IAAI,KAAK,EAAE,CAAC;QAChC,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,IAAI,CAAC,CAAC;QAC5B,MAAM,CAAC,GAAG,MAAM,MAAM,CAAC,GAAG,CAAC,OAAO,CAAC,QAAQ,EAAE,MAAM,EAAE,IAAI,EAAE,UAAU,EAAE,GAAG,GAAG,CAAC,EAAE;YAC9E,UAAU,EAAE,GAAG;SAChB,CAAC,CAAC;QACH,IAAI,CAAC,CAAC,QAAQ,KAAK,CAAC,IAAI,QAAQ,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC;YAC5C,OAAO;gBACL,EAAE,EAAE,KAAK;gBACT,KAAK,EAAE,kBAAkB,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,MAAM,GAAG,MAAM,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,IAAI,YAAY,8BAA8B;aAClH,CAAC;QACJ,CAAC;QACD,SAAS,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC;IACxB,CAAC;IACD,OAAO,EAAE,EAAE,EAAE,IAAI,EAAE,GAAG,EAAE,CAAC;AAC3B,CAAC;AAED;;;;;GAKG;AACH,MAAM,CAAC,KAAK,UAAU,aAAa,CACjC,QAAgB,EAChB,MAAkB;IAElB,KAAK,MAAM,MAAM,IAAI,CAAC,OAAO,EAAE,QAAQ,CAAU,EAAE,CAAC;QAClD,MAAM,CAAC,GAAG,MAAM,MAAM,CAAC,GAAG,CACxB,OAAO,CACL,QAAQ,EACR,KAAK,EACL,MAAM,EACN,WAAW,EACX,IAAI,EACJ,eAAe,CAAC,KAAK,EACrB,KAAK,EACL,OAAO,EACP,yCAAyC,EACzC,MAAM,CACP,CACF,CAAC;QACF,IAAI,CAAC,CAAC,QAAQ,KAAK,CAAC,EAAE,CAAC;YACrB,OAAO,EAAE,EAAE,EAAE,KAAK,EAAE,KAAK,EAAE,GAAG,MAAM,uBAAuB,IAAI,CAAC,CAAC,CAAC,MAAM,IAAI,CAAC,CAAC,MAAM,CAAC,EAAE,EAAE,CAAC;QAC5F,CAAC;IACH,CAAC;IACD,OAAO,EAAE,EAAE,EAAE,IAAI,EAAE,CAAC;AACtB,CAAC;AAED;;;;;GAKG;AACH,MAAM,YAAY,GAAwC;IACxD,KAAK,EACH,gIAAgI;IAClI,OAAO,EACL,oJAAoJ;CACvJ,CAAC;AAEF,KAAK,UAAU,WAAW,CACxB,OAA4B,EAC5B,QAAgB,EAChB,IAA2B;IAE3B,MAAM,QAAQ,GAAG,IAAI,CAAC,cAAc,IAAI,EAAE,CAAC;IAC3C,MAAM,QAAQ,GAAG,IAAI,CAAC,gBAAgB,IAAI,IAAI,CAAC;IAC/C,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,IAAI,CAAC,CAAC,EAAU,EAAE,EAAE,CAAC,IAAI,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,UAAU,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC;IACpF,IAAI,IAAI,GAAG,EAAE,CAAC;IACd,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,QAAQ,EAAE,CAAC,EAAE,EAAE,CAAC;QAClC,MAAM,CAAC,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,GAAG,CAC7B,OAAO,CAAC,QAAQ,EAAE,MAAM,EAAE,IAAI,EAAE,eAAe,CAAC,OAAO,CAAC,EAAE,KAAK,EAAE,IAAI,EAAE,YAAY,CAAC,OAAO,CAAC,CAAC,CAC9F,CAAC;QACF,IAAI,CAAC,CAAC,QAAQ,KAAK,CAAC;YAAE,OAAO,EAAE,EAAE,EAAE,IAAI,EAAE,CAAC;QAC1C,IAAI,GAAG,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC;QACtB,IAAI,CAAC,GAAG,QAAQ,GAAG,CAAC;YAAE,MAAM,KAAK,CAAC,QAAQ,CAAC,CAAC;IAC9C,CAAC;IACD,OAAO;QACL,EAAE,EAAE,KAAK;QACT,KAAK,EAAE,mCAAmC,QAAQ,YAAY,IAAI,CAAC,CAAC,CAAC,KAAK,IAAI,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE;KACzF,CAAC;AACJ,CAAC;AAED,0FAA0F;AAC1F,KAAK,UAAU,QAAQ,CACrB,IAAY,EACZ,eAAuB,EACvB,IAA2B;IAE3B,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,qCAAqC,CAAC,CAAC;IACrD,cAAc,CAAC,IAAI,EAAE,eAAe,CAAC,CAAC;IACtC,OAAO,CAAC,MAAM,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC;AACpC,CAAC;AAED,iFAAiF;AACjF,KAAK,UAAU,SAAS,CACtB,IAA2B;IAE3B,MAAM,EAAE,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,GAAG,CAC9B,OAAO,CAAC,IAAI,CAAC,QAAQ,EAAE,IAAI,EAAE,IAAI,EAAE,WAAW,EAAE,eAAe,CAAC,KAAK,EAAE,eAAe,CAAC,OAAO,CAAC,CAChG,CAAC;IACF,IAAI,EAAE,CAAC,QAAQ,KAAK,CAAC;QACnB,OAAO,EAAE,EAAE,EAAE,KAAK,EAAE,KAAK,EAAE,6BAA6B,IAAI,CAAC,EAAE,CAAC,MAAM,CAAC,EAAE,EAAE,CAAC;IAC9E,KAAK,MAAM,OAAO,IAAI,gBAAgB,EAAE,CAAC;QACvC,MAAM,OAAO,GAAG,MAAM,WAAW,CAAC,OAAO,EAAE,IAAI,CAAC,QAAQ,EAAE,IAAI,CAAC,CAAC;QAChE,IAAI,CAAC,OAAO,CAAC,EAAE;YAAE,OAAO,EAAE,EAAE,EAAE,KAAK,EAAE,KAAK,EAAE,GAAG,OAAO,KAAK,OAAO,CAAC,KAAK,EAAE,EAAE,CAAC;IAC/E,CAAC;IACD,OAAO,EAAE,EAAE,EAAE,IAAI,EAAE,CAAC;AACtB,CAAC;AAED,SAAS,cAAc,CAAC,IAAY,EAAE,QAAgB;IACpD,aAAa,CAAC,IAAI,EAAE,QAAQ,EAAE,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,CAAC;AACjD,CAAC;AAED,SAAS,WAAW,CAAC,GAAW;IAC9B,MAAM,EAAE,GAAG,GAAG,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC;IACnC,OAAO,EAAE,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,GAAG,CAAC,EAAE,EAAE,GAAG,EAAE,CAAC,CAAC;AACtD,CAAC;AAED,SAAS,IAAI,CAAC,CAAS;IACrB,OAAO,CAAC,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC;AACjE,CAAC"}
|