@wtfalch/foundry 0.38.0 → 0.39.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/bootstrap/versions.d.ts +3 -2
- package/dist/plan/backup-gate.d.ts +16 -0
- package/dist/plan/backup-gate.js +40 -0
- package/dist/plan/company-backups.js +15 -0
- package/dist/plan/diff.d.ts +4 -0
- package/dist/plan/diff.js +15 -1
- package/dist/plan/executor.d.ts +13 -0
- package/dist/plan/executor.js +49 -12
- package/dist/plan/own-diff.d.ts +26 -0
- package/dist/plan/own-diff.js +59 -2
- package/dist/plan/own-services.d.ts +10 -0
- package/dist/plan/own-services.js +10 -0
- package/dist/plan/plan-apply.d.ts +4 -0
- package/dist/plan/plan-apply.js +23 -1
- package/dist/plan/platform-providers.d.ts +46 -1
- package/dist/plan/platform-providers.js +79 -27
- package/dist/plan/signin-db-backup.d.ts +105 -0
- package/dist/plan/signin-db-backup.js +193 -0
- package/dist/plan/signin-db-move.d.ts +68 -0
- package/dist/plan/signin-db-move.js +338 -0
- package/dist/plan/signin-db-tls.d.ts +69 -0
- package/dist/plan/signin-db-tls.js +255 -0
- package/dist/plan/signin-db.d.ts +114 -0
- package/dist/plan/signin-db.js +255 -0
- package/package.json +1 -1
- package/versions.digests.json +2 -1
- package/versions.json +2 -1
|
@@ -0,0 +1,68 @@
|
|
|
1
|
+
import type { ProjectSource, ServiceTarget, StepScope } from './own-services.js';
|
|
2
|
+
import type { PlatformAuthDeps } from './platform-providers.js';
|
|
3
|
+
/**
|
|
4
|
+
* Moves a company's sign-in data from its Coolify database to the TLS database (#746).
|
|
5
|
+
*
|
|
6
|
+
* 1. The new database app is created and healthy (`provisionSigninDb`): ZITADEL is still up.
|
|
7
|
+
* 2. ZITADEL is stopped, so nothing writes to the old database.
|
|
8
|
+
* 3. A one-shot job app on the same Docker network empties the new database, runs `pg_dump` from
|
|
9
|
+
* the old internal URL into it, and compares the row count of every table in both. It reports
|
|
10
|
+
* healthy only after that passes, which is the signal the step waits for.
|
|
11
|
+
* 4. The CA is mounted on ZITADEL, its DSN is switched, the move is recorded, and ZITADEL starts.
|
|
12
|
+
*
|
|
13
|
+
* The old database is never written to and never deleted. After any failure that follows the stop,
|
|
14
|
+
* the step reads ZITADEL's DSN back from Coolify and acts on what is really set: the old DSN starts
|
|
15
|
+
* ZITADEL on the old database (the new app and the job app stay for inspection; a rerun copies again),
|
|
16
|
+
* the new DSN records the move and starts ZITADEL on the new database. A DSN write that timed out
|
|
17
|
+
* without a verdict, or a DSN that cannot be read, leaves ZITADEL stopped and says so. The move is
|
|
18
|
+
* recorded before ZITADEL starts on the new database, never after, and a rerun that finds the new DSN
|
|
19
|
+
* set finishes the move instead of copying, so it never drops a database ZITADEL may have written to.
|
|
20
|
+
*/
|
|
21
|
+
export declare const SIGNIN_DB_MOVE_ENV: {
|
|
22
|
+
readonly oldUrl: "OLD_DATABASE_URL";
|
|
23
|
+
readonly newUrl: "NEW_DATABASE_URL";
|
|
24
|
+
readonly newAdminUrl: "NEW_ADMIN_DATABASE_URL";
|
|
25
|
+
readonly ca: "NEW_DATABASE_CA_B64";
|
|
26
|
+
};
|
|
27
|
+
/** Recorded id of the copy job app. */
|
|
28
|
+
export declare const SIGNIN_DB_MOVE_APP_SCOPE_KEY = "signin_db_move_app";
|
|
29
|
+
/**
|
|
30
|
+
* The job's script. Every line is plain shell; the Dockerfile carries it base64-encoded, so the SQL
|
|
31
|
+
* may hold quotes. It prints table counts and its own messages, never a URL.
|
|
32
|
+
*/
|
|
33
|
+
export declare function signinDbMoveScript(): string[];
|
|
34
|
+
/** The copy job: the same pinned image as the database, so `pg_dump` matches the server's major version. */
|
|
35
|
+
export declare function signinDbMoveDockerfile(): string;
|
|
36
|
+
/** A URL the job connects to the new database with; the host name is checked against the certificate. */
|
|
37
|
+
export declare function signinDbMoveNewUrl(host: string, password: string, database: string): string;
|
|
38
|
+
export interface MoveRuntime {
|
|
39
|
+
sleep?: (ms: number) => Promise<void>;
|
|
40
|
+
now?: () => number;
|
|
41
|
+
onSecret?: (value: string) => void;
|
|
42
|
+
/** How long the copy job may run (default 30 minutes). */
|
|
43
|
+
signinDbMoveTimeoutMs?: number;
|
|
44
|
+
/** Pause between polls of the job (default 10 seconds). */
|
|
45
|
+
signinDbMovePollMs?: number;
|
|
46
|
+
/** How long ZITADEL may take to report stopped before the copy (default 2 minutes). */
|
|
47
|
+
signinDbStopTimeoutMs?: number;
|
|
48
|
+
/** How long ZITADEL may take to report healthy on the new database (default 5 minutes). */
|
|
49
|
+
readyTimeoutMs?: number;
|
|
50
|
+
/** How long the new database may take to report healthy (default 5 minutes). */
|
|
51
|
+
signinDbTimeoutMs?: number;
|
|
52
|
+
}
|
|
53
|
+
export interface MoveInput {
|
|
54
|
+
project: ProjectSource;
|
|
55
|
+
label: string;
|
|
56
|
+
}
|
|
57
|
+
/**
|
|
58
|
+
* The step. Returns without doing anything once the move is recorded. See the file comment for the
|
|
59
|
+
* order and for what a failure at each stage leaves behind.
|
|
60
|
+
*/
|
|
61
|
+
export declare function moveSigninDatabase(scope: StepScope, deps: Pick<PlatformAuthDeps, 'coolify' | 'heldKeys' | 'signinDbBackup'> & {
|
|
62
|
+
runtime?: MoveRuntime;
|
|
63
|
+
/** Reads the old database without the resource-limits patch the limited port adds: the old database is only read. */
|
|
64
|
+
readOldDatabase(uuid: string): Promise<{
|
|
65
|
+
internal_db_url?: string;
|
|
66
|
+
status?: string;
|
|
67
|
+
}>;
|
|
68
|
+
}, target: ServiceTarget, input: MoveInput): Promise<void>;
|
|
@@ -0,0 +1,338 @@
|
|
|
1
|
+
import { pinnedRef } from '../bootstrap/versions.js';
|
|
2
|
+
import { PLATFORM_AUTH_DB_SCOPE_KEY, PLATFORM_AUTH_ZITADEL_SCOPE_KEY, findOrCreatePlatformApp, } from './platform-providers.js';
|
|
3
|
+
import { SIGNIN_DB_TLS_SCOPE_KEY, SIGNIN_DB_ZITADEL_PENDING_SCOPE_KEY } from './signin-db-tls.js';
|
|
4
|
+
import { SIGNIN_DB_NAME, SIGNIN_DB_PORT, SIGNIN_DB_USER, boundedCall, dockerfileAppCreator, ensureSigninDbCaFile, provisionSigninDb, waitHealthy, withCallTimeout, } from './signin-db.js';
|
|
5
|
+
/**
|
|
6
|
+
* Moves a company's sign-in data from its Coolify database to the TLS database (#746).
|
|
7
|
+
*
|
|
8
|
+
* 1. The new database app is created and healthy (`provisionSigninDb`): ZITADEL is still up.
|
|
9
|
+
* 2. ZITADEL is stopped, so nothing writes to the old database.
|
|
10
|
+
* 3. A one-shot job app on the same Docker network empties the new database, runs `pg_dump` from
|
|
11
|
+
* the old internal URL into it, and compares the row count of every table in both. It reports
|
|
12
|
+
* healthy only after that passes, which is the signal the step waits for.
|
|
13
|
+
* 4. The CA is mounted on ZITADEL, its DSN is switched, the move is recorded, and ZITADEL starts.
|
|
14
|
+
*
|
|
15
|
+
* The old database is never written to and never deleted. After any failure that follows the stop,
|
|
16
|
+
* the step reads ZITADEL's DSN back from Coolify and acts on what is really set: the old DSN starts
|
|
17
|
+
* ZITADEL on the old database (the new app and the job app stay for inspection; a rerun copies again),
|
|
18
|
+
* the new DSN records the move and starts ZITADEL on the new database. A DSN write that timed out
|
|
19
|
+
* without a verdict, or a DSN that cannot be read, leaves ZITADEL stopped and says so. The move is
|
|
20
|
+
* recorded before ZITADEL starts on the new database, never after, and a rerun that finds the new DSN
|
|
21
|
+
* set finishes the move instead of copying, so it never drops a database ZITADEL may have written to.
|
|
22
|
+
*/
|
|
23
|
+
export const SIGNIN_DB_MOVE_ENV = {
|
|
24
|
+
oldUrl: 'OLD_DATABASE_URL',
|
|
25
|
+
newUrl: 'NEW_DATABASE_URL',
|
|
26
|
+
newAdminUrl: 'NEW_ADMIN_DATABASE_URL',
|
|
27
|
+
ca: 'NEW_DATABASE_CA_B64',
|
|
28
|
+
};
|
|
29
|
+
/** Where the job writes the CA it is given in env. */
|
|
30
|
+
const JOB_CA_PATH = '/tmp/ca.crt';
|
|
31
|
+
const JOB_DONE_FILE = '/tmp/done';
|
|
32
|
+
/** Left by a run that did not finish: a restart of the same container then stops at once instead of copying again. */
|
|
33
|
+
const JOB_FAILED_FILE = '/tmp/failed';
|
|
34
|
+
const JOB_DUMP_FILE = '/tmp/dump.sql';
|
|
35
|
+
/** Recorded id of the copy job app. */
|
|
36
|
+
export const SIGNIN_DB_MOVE_APP_SCOPE_KEY = 'signin_db_move_app';
|
|
37
|
+
/**
|
|
38
|
+
* The job's script. Every line is plain shell; the Dockerfile carries it base64-encoded, so the SQL
|
|
39
|
+
* may hold quotes. It prints table counts and its own messages, never a URL.
|
|
40
|
+
*/
|
|
41
|
+
export function signinDbMoveScript() {
|
|
42
|
+
const old = `"$${SIGNIN_DB_MOVE_ENV.oldUrl}"`;
|
|
43
|
+
const next = `"$${SIGNIN_DB_MOVE_ENV.newUrl}"`;
|
|
44
|
+
const admin = `"$${SIGNIN_DB_MOVE_ENV.newAdminUrl}"`;
|
|
45
|
+
const tables = "select quote_ident(schemaname) || '.' || quote_ident(tablename) from pg_tables where schemaname not in ('pg_catalog', 'information_schema') order by 1";
|
|
46
|
+
return [
|
|
47
|
+
'#!/bin/sh',
|
|
48
|
+
'set -eu',
|
|
49
|
+
// The dump holds the whole sign-in database: it is removed on every exit. A run that ends without
|
|
50
|
+
// `done` leaves `failed`, because Coolify restarts the container and a restart must not copy again.
|
|
51
|
+
`trap 'rm -f ${JOB_DUMP_FILE}; [ -f ${JOB_DONE_FILE} ] || touch ${JOB_FAILED_FILE}' EXIT`,
|
|
52
|
+
"trap 'exit 1' HUP INT TERM",
|
|
53
|
+
`[ ! -f ${JOB_FAILED_FILE} ] || exit 1`,
|
|
54
|
+
`rm -f ${JOB_DONE_FILE}`,
|
|
55
|
+
`printf %s "$${SIGNIN_DB_MOVE_ENV.ca}" | base64 -d > ${JOB_CA_PATH}`,
|
|
56
|
+
'fail() { echo "signin-db-move: $1"; exit 1; }',
|
|
57
|
+
`major=$(psql ${old} -Atc "show server_version_num" | cut -c1-2)`,
|
|
58
|
+
'[ "$major" = "16" ] || fail "the old database is not Postgres 16"',
|
|
59
|
+
`psql ${admin} -v ON_ERROR_STOP=1 -q -c "drop database if exists ${SIGNIN_DB_NAME} with (force)" -c "create database ${SIGNIN_DB_NAME}" || fail "the new database could not be emptied"`,
|
|
60
|
+
`pg_dump ${old} > ${JOB_DUMP_FILE} || fail "pg_dump failed"`,
|
|
61
|
+
`psql ${next} -v ON_ERROR_STOP=1 -q -f ${JOB_DUMP_FILE} > /dev/null || fail "the restore failed"`,
|
|
62
|
+
`rm -f ${JOB_DUMP_FILE}`,
|
|
63
|
+
`old_tables=$(psql ${old} -Atc "${tables}")`,
|
|
64
|
+
`new_tables=$(psql ${next} -Atc "${tables}")`,
|
|
65
|
+
'[ "$old_tables" = "$new_tables" ] || fail "the two databases hold different tables"',
|
|
66
|
+
'checked=0',
|
|
67
|
+
'for t in $old_tables; do',
|
|
68
|
+
` a=$(psql ${old} -Atc "select count(*) from $t")`,
|
|
69
|
+
` b=$(psql ${next} -Atc "select count(*) from $t")`,
|
|
70
|
+
' [ "$a" = "$b" ] || fail "row count differs in $t: old $a, new $b"',
|
|
71
|
+
' checked=$((checked + 1))',
|
|
72
|
+
'done',
|
|
73
|
+
'[ "$checked" -gt 0 ] || fail "the old database has no tables"',
|
|
74
|
+
'echo "signin-db-move: $checked tables copied, row counts equal"',
|
|
75
|
+
`touch ${JOB_DONE_FILE}`,
|
|
76
|
+
'exec sleep infinity',
|
|
77
|
+
];
|
|
78
|
+
}
|
|
79
|
+
/** The copy job: the same pinned image as the database, so `pg_dump` matches the server's major version. */
|
|
80
|
+
export function signinDbMoveDockerfile() {
|
|
81
|
+
const encoded = Buffer.from(`${signinDbMoveScript().join('\n')}\n`, 'utf8').toString('base64');
|
|
82
|
+
return [
|
|
83
|
+
`FROM postgres:${pinnedRef('postgres')}`,
|
|
84
|
+
`RUN echo '${encoded}' | base64 -d > /usr/local/bin/signin-db-move && chmod 0755 /usr/local/bin/signin-db-move`,
|
|
85
|
+
'ENTRYPOINT ["/usr/local/bin/signin-db-move"]',
|
|
86
|
+
`HEALTHCHECK --interval=10s --timeout=5s --start-period=10s --retries=1 CMD ["test", "-f", "${JOB_DONE_FILE}"]`,
|
|
87
|
+
].join('\n');
|
|
88
|
+
}
|
|
89
|
+
/** A URL the job connects to the new database with; the host name is checked against the certificate. */
|
|
90
|
+
export function signinDbMoveNewUrl(host, password, database) {
|
|
91
|
+
return `postgres://${SIGNIN_DB_USER}:${password}@${host}:${SIGNIN_DB_PORT}/${database}?sslmode=verify-full&sslrootcert=${JOB_CA_PATH}`;
|
|
92
|
+
}
|
|
93
|
+
/** A failure the step words itself. Anything else that goes wrong is reported without its cause, which could carry a URL. */
|
|
94
|
+
class MoveFailure extends Error {
|
|
95
|
+
}
|
|
96
|
+
const defaultSleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
|
|
97
|
+
/** Polls of an `exited` job with no `running` seen before it counts as a stop (an `exited` read can be the container before the deploy). */
|
|
98
|
+
const JOB_EXITED_POLLS = 6;
|
|
99
|
+
/**
|
|
100
|
+
* Waits for the copy job to report healthy, which it does only after the counts match. The job never
|
|
101
|
+
* exits on success, so any exit is final: `restarting` at once, `exited` once the job was seen running
|
|
102
|
+
* or stayed exited for {@link JOB_EXITED_POLLS} polls. It has failed too at the deadline. A healthy
|
|
103
|
+
* status read before any other could be the container before the stop, so it counts only after a
|
|
104
|
+
* different status was seen.
|
|
105
|
+
*/
|
|
106
|
+
async function waitForJob(coolify, appId, runtime) {
|
|
107
|
+
const now = runtime.now ?? Date.now;
|
|
108
|
+
const sleep = runtime.sleep ?? defaultSleep;
|
|
109
|
+
const timeout = runtime.signinDbMoveTimeoutMs ?? 1_800_000;
|
|
110
|
+
const interval = runtime.signinDbMovePollMs ?? 10_000;
|
|
111
|
+
const deadline = now() + timeout;
|
|
112
|
+
let left = false;
|
|
113
|
+
let running = false;
|
|
114
|
+
let exitedPolls = 0;
|
|
115
|
+
for (;;) {
|
|
116
|
+
const { status } = await coolify.getApplication(appId);
|
|
117
|
+
if (status === 'running:healthy' && left)
|
|
118
|
+
return;
|
|
119
|
+
if (status !== 'running:healthy')
|
|
120
|
+
left = true;
|
|
121
|
+
if (status?.startsWith('running'))
|
|
122
|
+
running = true;
|
|
123
|
+
exitedPolls = status?.startsWith('exited') ? exitedPolls + 1 : 0;
|
|
124
|
+
if (status?.startsWith('restarting') ||
|
|
125
|
+
(status?.startsWith('exited') && (running || exitedPolls >= JOB_EXITED_POLLS))) {
|
|
126
|
+
throw new MoveFailure('the copy job stopped before the row counts matched');
|
|
127
|
+
}
|
|
128
|
+
if (now() >= deadline) {
|
|
129
|
+
throw new MoveFailure(`the copy job did not finish within ${Math.round(timeout / 1000)}s (last status: ${status ?? 'unknown'})`);
|
|
130
|
+
}
|
|
131
|
+
await sleep(interval);
|
|
132
|
+
}
|
|
133
|
+
}
|
|
134
|
+
/** Coolify queues a stop. Waits until it reports the app stopped; `pg_dump` must not run while ZITADEL can still write. */
|
|
135
|
+
async function waitStopped(coolify, appId, runtime) {
|
|
136
|
+
const now = runtime.now ?? Date.now;
|
|
137
|
+
const sleep = runtime.sleep ?? defaultSleep;
|
|
138
|
+
const timeout = runtime.signinDbStopTimeoutMs ?? 120_000;
|
|
139
|
+
const interval = runtime.signinDbMovePollMs ?? 10_000;
|
|
140
|
+
const deadline = now() + timeout;
|
|
141
|
+
for (;;) {
|
|
142
|
+
const { status } = await coolify.getApplication(appId);
|
|
143
|
+
if (status?.startsWith('exited') || status?.startsWith('stopped'))
|
|
144
|
+
return;
|
|
145
|
+
if (now() >= deadline) {
|
|
146
|
+
throw new MoveFailure(`ZITADEL did not report stopped within ${Math.round(timeout / 1000)}s (last status: ${status ?? 'unknown'}), so nothing was copied`);
|
|
147
|
+
}
|
|
148
|
+
await sleep(interval);
|
|
149
|
+
}
|
|
150
|
+
}
|
|
151
|
+
/** Runs `work`; a failure comes back as a note for the error message, never thrown. */
|
|
152
|
+
async function attempt(work, failed) {
|
|
153
|
+
try {
|
|
154
|
+
await work();
|
|
155
|
+
return '';
|
|
156
|
+
}
|
|
157
|
+
catch (error) {
|
|
158
|
+
return error instanceof MoveFailure ? ` (${failed}: ${error.message})` : ` (${failed})`;
|
|
159
|
+
}
|
|
160
|
+
}
|
|
161
|
+
/**
|
|
162
|
+
* The step. Returns without doing anything once the move is recorded. See the file comment for the
|
|
163
|
+
* order and for what a failure at each stage leaves behind.
|
|
164
|
+
*/
|
|
165
|
+
export async function moveSigninDatabase(scope, deps, target, input) {
|
|
166
|
+
const coolify = withCallTimeout(deps.coolify);
|
|
167
|
+
const { label } = input;
|
|
168
|
+
const runtime = deps.runtime ?? {};
|
|
169
|
+
const recorded = scope.recordedIds();
|
|
170
|
+
if (recorded[SIGNIN_DB_TLS_SCOPE_KEY])
|
|
171
|
+
return;
|
|
172
|
+
const oldDbId = recorded[PLATFORM_AUTH_DB_SCOPE_KEY];
|
|
173
|
+
const zitadelId = recorded[PLATFORM_AUTH_ZITADEL_SCOPE_KEY];
|
|
174
|
+
if (!oldDbId || !zitadelId) {
|
|
175
|
+
throw new Error(`${label}: the sign-in database cannot be moved: the company's sign-in is not applied yet`);
|
|
176
|
+
}
|
|
177
|
+
const record = async (key, id) => {
|
|
178
|
+
scope.recordScopeId(key, id);
|
|
179
|
+
await scope.persist();
|
|
180
|
+
};
|
|
181
|
+
const old = await boundedCall(deps.readOldDatabase(oldDbId), 'Coolify getDatabase');
|
|
182
|
+
if (!old.internal_db_url || !old.status?.startsWith('running')) {
|
|
183
|
+
throw new Error(`${label}: the sign-in database cannot be moved: the old database is not running`);
|
|
184
|
+
}
|
|
185
|
+
runtime.onSecret?.(old.internal_db_url);
|
|
186
|
+
// The password alone too: a message that quotes only that part must be scrubbed as well.
|
|
187
|
+
if (URL.canParse(old.internal_db_url)) {
|
|
188
|
+
const password = decodeURIComponent(new URL(old.internal_db_url).password);
|
|
189
|
+
if (password)
|
|
190
|
+
runtime.onSecret?.(password);
|
|
191
|
+
}
|
|
192
|
+
const projectUuid = await input.project.ensure(coolify, scope);
|
|
193
|
+
const { environment } = await coolify.findOrCreateEnvironment(projectUuid, 'production');
|
|
194
|
+
const creator = (name, description) => dockerfileAppCreator(coolify, { projectUuid, environment, name, description });
|
|
195
|
+
// (b) The new database. ZITADEL is still up on the old one.
|
|
196
|
+
const db = await provisionSigninDb({
|
|
197
|
+
coolify,
|
|
198
|
+
keys: deps.heldKeys,
|
|
199
|
+
runtime,
|
|
200
|
+
name: target.names.signinDb,
|
|
201
|
+
internalHost: target.internal.signinDb,
|
|
202
|
+
volumeName: `${target.names.signinDb}-data`,
|
|
203
|
+
serverCertRef: target.refs.signinDbServer,
|
|
204
|
+
slug: target.id,
|
|
205
|
+
backup: deps.signinDbBackup,
|
|
206
|
+
backupBucket: target.names.signinBackupBucket,
|
|
207
|
+
recorded,
|
|
208
|
+
label,
|
|
209
|
+
environment,
|
|
210
|
+
record,
|
|
211
|
+
create: creator(target.names.signinDb, `${target.id}'s sign-in database with TLS (#746)`),
|
|
212
|
+
});
|
|
213
|
+
const newUrl = signinDbMoveNewUrl(target.internal.signinDb, db.password, SIGNIN_DB_NAME);
|
|
214
|
+
const adminUrl = signinDbMoveNewUrl(target.internal.signinDb, db.password, 'postgres');
|
|
215
|
+
runtime.onSecret?.(newUrl);
|
|
216
|
+
runtime.onSecret?.(adminUrl);
|
|
217
|
+
const dsnKey = 'ZITADEL_DATABASE_POSTGRES_DSN';
|
|
218
|
+
/** Which database ZITADEL's DSN names in Coolify right now. */
|
|
219
|
+
const liveDatabase = async () => {
|
|
220
|
+
try {
|
|
221
|
+
const dsn = (await coolify.listEnvs(zitadelId)).find((e) => e.key === dsnKey && !e.is_preview)?.value;
|
|
222
|
+
if (dsn === db.dsn)
|
|
223
|
+
return 'new';
|
|
224
|
+
return dsn === old.internal_db_url ? 'old' : 'unknown';
|
|
225
|
+
}
|
|
226
|
+
catch {
|
|
227
|
+
// The cause may carry a URL; the caller words the state.
|
|
228
|
+
return 'unknown';
|
|
229
|
+
}
|
|
230
|
+
};
|
|
231
|
+
/**
|
|
232
|
+
* ZITADEL's DSN names the new database. The move is recorded, with the flag that ZITADEL is not yet
|
|
233
|
+
* seen healthy there (it makes the planner plan the sign-in step), in one write before ZITADEL
|
|
234
|
+
* starts. Every failure says the exact state.
|
|
235
|
+
*/
|
|
236
|
+
const startOnNew = async () => {
|
|
237
|
+
scope.recordScopeId(SIGNIN_DB_TLS_SCOPE_KEY, db.appId);
|
|
238
|
+
scope.recordScopeId(SIGNIN_DB_ZITADEL_PENDING_SCOPE_KEY, '1');
|
|
239
|
+
try {
|
|
240
|
+
await scope.persist();
|
|
241
|
+
}
|
|
242
|
+
catch {
|
|
243
|
+
throw new MoveFailure('ZITADEL is stopped and its DSN names the new database; the move could not be recorded, so ZITADEL was not started on it; apply again, which records the move and starts it');
|
|
244
|
+
}
|
|
245
|
+
try {
|
|
246
|
+
await coolify.deploy(zitadelId);
|
|
247
|
+
}
|
|
248
|
+
catch {
|
|
249
|
+
throw new MoveFailure('the move is recorded and ZITADEL uses the new database, but it could not be started; start it in Coolify (the next apply starts it too)');
|
|
250
|
+
}
|
|
251
|
+
try {
|
|
252
|
+
await waitHealthy(coolify, zitadelId, { ...runtime, signinDbTimeoutMs: runtime.readyTimeoutMs ?? 300_000 }, label, 'ZITADEL');
|
|
253
|
+
}
|
|
254
|
+
catch (error) {
|
|
255
|
+
// waitHealthy words its own message (the app, the status); nothing else is repeated.
|
|
256
|
+
const why = error instanceof Error && error.message.includes('did not report healthy')
|
|
257
|
+
? ''
|
|
258
|
+
: ' (a Coolify call failed)';
|
|
259
|
+
throw new MoveFailure(`ZITADEL did not report healthy on the new sign-in database${why}; the move is recorded, the old database is untouched, check ZITADEL's logs and apply again`);
|
|
260
|
+
}
|
|
261
|
+
await record(SIGNIN_DB_ZITADEL_PENDING_SCOPE_KEY, '');
|
|
262
|
+
};
|
|
263
|
+
// An earlier try that got as far as the DSN switch: ZITADEL may have written to the new database,
|
|
264
|
+
// so it is never emptied again. The move is finished instead.
|
|
265
|
+
const finished = (error) => {
|
|
266
|
+
throw new Error(`${label}: ${error instanceof MoveFailure ? error.message : 'the move could not be finished'}`);
|
|
267
|
+
};
|
|
268
|
+
const before = await liveDatabase();
|
|
269
|
+
if (before === 'new') {
|
|
270
|
+
await startOnNew().catch(finished);
|
|
271
|
+
return;
|
|
272
|
+
}
|
|
273
|
+
if (before === 'unknown') {
|
|
274
|
+
throw new Error(`${label}: the sign-in database cannot be moved: ZITADEL's DSN could not be read or names neither database, so nothing is copied; ZITADEL may still be stopped from an earlier attempt and this apply does not start it: read the DSN in Coolify, set it to one of the two databases, then apply again`);
|
|
275
|
+
}
|
|
276
|
+
let dsnWriteStarted = false;
|
|
277
|
+
let jobAppId;
|
|
278
|
+
try {
|
|
279
|
+
// (c) Nothing writes to the old database from here, once Coolify reports ZITADEL stopped.
|
|
280
|
+
await coolify.raw('POST', `/applications/${zitadelId}/stop`);
|
|
281
|
+
await waitStopped(coolify, zitadelId, runtime);
|
|
282
|
+
// (d) The copy job.
|
|
283
|
+
const job = await findOrCreatePlatformApp(coolify, target.names.signinDbMove, environment, () => creator(target.names.signinDbMove, `${target.id}'s sign-in database copy job (#746)`)(signinDbMoveDockerfile()));
|
|
284
|
+
jobAppId = job.appId;
|
|
285
|
+
await record(SIGNIN_DB_MOVE_APP_SCOPE_KEY, job.appId);
|
|
286
|
+
await coolify.updateApplication(job.appId, { health_check_enabled: false });
|
|
287
|
+
await coolify.bulkUpsertEnvs(job.appId, [
|
|
288
|
+
{ key: SIGNIN_DB_MOVE_ENV.oldUrl, value: old.internal_db_url },
|
|
289
|
+
{ key: SIGNIN_DB_MOVE_ENV.newUrl, value: newUrl },
|
|
290
|
+
{ key: SIGNIN_DB_MOVE_ENV.newAdminUrl, value: adminUrl },
|
|
291
|
+
{ key: SIGNIN_DB_MOVE_ENV.ca, value: Buffer.from(db.ca, 'utf8').toString('base64') },
|
|
292
|
+
]);
|
|
293
|
+
// A job left running by a failed run would still read as healthy or as running.
|
|
294
|
+
if (job.running)
|
|
295
|
+
await coolify.raw('POST', `/applications/${job.appId}/stop`);
|
|
296
|
+
await coolify.deploy(job.appId, true);
|
|
297
|
+
await waitForJob(coolify, job.appId, runtime);
|
|
298
|
+
await coolify.raw('POST', `/applications/${job.appId}/stop`);
|
|
299
|
+
// (e) ZITADEL's side of the switch. The write may land even when the call does not answer.
|
|
300
|
+
await ensureSigninDbCaFile(coolify, zitadelId, db.ca, label);
|
|
301
|
+
dsnWriteStarted = true;
|
|
302
|
+
await coolify.bulkUpsertEnvs(zitadelId, [{ key: dsnKey, value: db.dsn }]);
|
|
303
|
+
}
|
|
304
|
+
catch (error) {
|
|
305
|
+
const reason = error instanceof MoveFailure
|
|
306
|
+
? error.message
|
|
307
|
+
: 'a Coolify call failed (its message is not repeated)';
|
|
308
|
+
const timedOut = error instanceof Error && error.message.includes('did not answer within');
|
|
309
|
+
let state;
|
|
310
|
+
let tail = 'the old database is untouched and the new one is left for inspection';
|
|
311
|
+
const live = await liveDatabase();
|
|
312
|
+
if (live === 'new') {
|
|
313
|
+
tail = 'the old database is untouched';
|
|
314
|
+
try {
|
|
315
|
+
await startOnNew();
|
|
316
|
+
state = 'ZITADEL was started on the NEW database and reports healthy; the move is recorded';
|
|
317
|
+
}
|
|
318
|
+
catch (startError) {
|
|
319
|
+
state = `ZITADEL uses the NEW database, but ${startError instanceof MoveFailure ? startError.message : 'it could not be started'}`;
|
|
320
|
+
}
|
|
321
|
+
}
|
|
322
|
+
else if (live === 'old' && !(dsnWriteStarted && timedOut)) {
|
|
323
|
+
state = `ZITADEL was started again on the old database${await attempt(() => coolify.deploy(zitadelId), 'ZITADEL could not be restarted, start it by hand')}`;
|
|
324
|
+
}
|
|
325
|
+
else {
|
|
326
|
+
state =
|
|
327
|
+
live === 'old'
|
|
328
|
+
? 'ZITADEL is left stopped: its DSN still names the old database, but the write of the new one did not answer and may still land; do not start ZITADEL by hand, because writes on the old database would be lost if the new DSN lands later; wait until its DSN reads the same twice a minute apart in Coolify, then apply again'
|
|
329
|
+
: 'ZITADEL is left stopped: its DSN could not be read or names neither database; do not start ZITADEL by hand; set its DSN to the old database in Coolify, then apply again';
|
|
330
|
+
}
|
|
331
|
+
const stoppedJob = jobAppId
|
|
332
|
+
? await attempt(() => coolify.raw('POST', `/applications/${jobAppId}/stop`), 'the copy job could not be stopped')
|
|
333
|
+
: '';
|
|
334
|
+
throw new Error(`${label}: moving the sign-in database failed: ${reason}. ${state}${stoppedJob}; ${tail}`);
|
|
335
|
+
}
|
|
336
|
+
// Recorded before ZITADEL starts on the new database, so a rerun never copies over what it writes there.
|
|
337
|
+
await startOnNew().catch(finished);
|
|
338
|
+
}
|
|
@@ -0,0 +1,69 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The certificate authority and server certificate of a company's sign-in
|
|
3
|
+
* database (#746). One CA serves the whole platform and is created once; each
|
|
4
|
+
* company's database gets its own server certificate, signed by it, for the
|
|
5
|
+
* database's internal hostname. Both are held in `@wtfalch/keys` as one JSON
|
|
6
|
+
* value each (`{ cert, key }`), so a CA is never half written.
|
|
7
|
+
*
|
|
8
|
+
* Node has no certificate builder, so the DER is written here: ECDSA P-256,
|
|
9
|
+
* SHA-256, X.509 v3, only the extensions Postgres clients check.
|
|
10
|
+
*/
|
|
11
|
+
/** The platform's one CA, held as JSON `{ cert, key }` (PEM strings). */
|
|
12
|
+
export declare const SIGNIN_DB_CA_REF = "signin-db:platform:ca";
|
|
13
|
+
/** The recorded id of the server certificate's expiry (milliseconds since the epoch), so the planner can renew it. Not a secret. */
|
|
14
|
+
export declare const SIGNIN_DB_CERT_NOT_AFTER_SCOPE_KEY = "signin_db_cert_not_after";
|
|
15
|
+
/** Recorded id of the database app. */
|
|
16
|
+
export declare const SIGNIN_DB_APP_SCOPE_KEY = "signin_db_app";
|
|
17
|
+
/**
|
|
18
|
+
* Recorded once ZITADEL's DSN names the TLS database: from then on the sign-in step
|
|
19
|
+
* provisions this database and never the Coolify one.
|
|
20
|
+
*/
|
|
21
|
+
export declare const SIGNIN_DB_TLS_SCOPE_KEY = "signin_db_tls";
|
|
22
|
+
/**
|
|
23
|
+
* Recorded while ZITADEL has been started on the TLS database and not yet seen healthy there: the
|
|
24
|
+
* planner then plans the sign-in step, which starts ZITADEL again and clears it (#746).
|
|
25
|
+
*/
|
|
26
|
+
export declare const SIGNIN_DB_ZITADEL_PENDING_SCOPE_KEY = "signin_db_zitadel_pending";
|
|
27
|
+
/** A server certificate with less than this left is issued again. */
|
|
28
|
+
export declare const SIGNIN_DB_CERT_RENEW_WINDOW_MS: number;
|
|
29
|
+
export interface SigninDbKeys {
|
|
30
|
+
resolve(ref: string): Promise<string | null | undefined>;
|
|
31
|
+
put?(ref: string, value: string): Promise<void>;
|
|
32
|
+
}
|
|
33
|
+
export interface SigninDbTlsRuntime {
|
|
34
|
+
/** Called with every secret before it is stored or used, so no message or audit row can carry it. */
|
|
35
|
+
onSecret?: (value: string) => void;
|
|
36
|
+
now?: () => number;
|
|
37
|
+
}
|
|
38
|
+
export interface PemPair {
|
|
39
|
+
certPem: string;
|
|
40
|
+
keyPem: string;
|
|
41
|
+
}
|
|
42
|
+
export interface ServerCert extends PemPair {
|
|
43
|
+
/** Milliseconds since the epoch. */
|
|
44
|
+
notAfter: number;
|
|
45
|
+
/** True when this call made a new certificate. */
|
|
46
|
+
issued: boolean;
|
|
47
|
+
}
|
|
48
|
+
/**
|
|
49
|
+
* The platform's CA: the held one, else a new one stored under
|
|
50
|
+
* {@link SIGNIN_DB_CA_REF}. A held value that cannot be read, or whose
|
|
51
|
+
* certificate does not match its key, is an error and is never replaced: a
|
|
52
|
+
* company's database may already trust it. The value read back after a store
|
|
53
|
+
* is the one used, so two applies at once end with the same CA.
|
|
54
|
+
*/
|
|
55
|
+
export declare function ensureSigninDbCa(keys: SigninDbKeys, runtime?: SigninDbTlsRuntime,
|
|
56
|
+
/** A company already has a database or certificate made under the CA: a lost CA is then an error, never made again. */
|
|
57
|
+
inUse?: boolean): Promise<PemPair>;
|
|
58
|
+
/**
|
|
59
|
+
* The server certificate held under `ref` for `host`, issued again when it is
|
|
60
|
+
* missing, unreadable, for another host, signed by another CA, or has less
|
|
61
|
+
* than {@link SIGNIN_DB_CERT_RENEW_WINDOW_MS} left. The read-back after a
|
|
62
|
+
* store is the one used.
|
|
63
|
+
*/
|
|
64
|
+
export declare function ensureSigninDbServerCert(keys: SigninDbKeys, ca: PemPair, ref: string, host: string, runtime?: SigninDbTlsRuntime): Promise<ServerCert>;
|
|
65
|
+
/**
|
|
66
|
+
* Whether the recorded server certificate expires within the renew window.
|
|
67
|
+
* Nothing recorded (a company not on the TLS database) is never due.
|
|
68
|
+
*/
|
|
69
|
+
export declare function signinDbCertDue(recordedIds: Readonly<Record<string, string>>, now: number): boolean;
|