@wtfalch/foundry 0.37.0 → 0.39.0

package/dist/plan/signin-db-backup.js

@@ -0,0 +1,193 @@
+import { SIGNIN_DB_TLS_SCOPE_KEY } from './signin-db-tls.js';
+/**
+ * The nightly backup of a company's sign-in database (#746). Coolify's database backups cover Coolify
+ * databases only, and the TLS database is an application, so the schedule is a Coolify scheduled task of
+ * that application (`POST /applications/{uuid}/scheduled-tasks`, read from Coolify 4.4.2's
+ * `ScheduledTasksController`). Coolify runs the task's command with `docker exec` in the database's
+ * container at the task's frequency. The command dumps with `pg_dump` and puts the file in the company's
+ * own backup bucket, with a storage key that reaches that bucket alone (minted through the same host
+ * ports as the company's files bucket key, `ensureSigninDbBackupTarget`). The platform's backup key never
+ * reaches a company's container. It keeps nothing: Coolify's default of keeping every dump applies.
+ */
+/** Recorded id of the scheduled task (set only after Coolify lists it back as enabled). */
+export const SIGNIN_DB_BACKUP_TASK_SCOPE_KEY = 'signin_db_backup_task';
+export const SIGNIN_DB_BACKUP_TASK_NAME = 'signin-db-backup';
+/** Coolify's own name for "once a day", the frequency the platform's database backups use. */
+export const SIGNIN_DB_BACKUP_FREQUENCY = 'daily';
+/** How long Coolify lets one run take, in seconds. */
+export const SIGNIN_DB_BACKUP_TIMEOUT_SECONDS = 3600;
+/** How long the upload may take, in seconds: the dump has to leave time inside the task's own limit. */
+const UPLOAD_MAX_SECONDS = 1800;
+/** The token id of the company's sign-in backup key, recorded once it is in the database app's env. */
+export const SIGNIN_DB_BACKUP_KEY_SCOPE_KEY = 'signin_db_backup_key';
+/** The token id minted and not yet in the env: deleted before any new mint. */
+export const SIGNIN_DB_BACKUP_KEY_UNDELIVERED_SCOPE_KEY = 'signin_db_backup_key_undelivered';
+/** How long one call to the bucket or key port may take. */
+export const SIGNIN_DB_BACKUP_PORT_TIMEOUT_MS = 30_000;
+async function portCall(work, label, what) {
+    let timer;
+    try {
+        return await Promise.race([
+            work,
+            new Promise((_, reject) => {
+                timer = setTimeout(() => reject(new Error('timeout')), SIGNIN_DB_BACKUP_PORT_TIMEOUT_MS);
+            }),
+        ]);
+    }
+    catch (error) {
+        // The cause is dropped: a port's message is not ours to repeat, and may hold the secret.
+        throw new Error(`${label}: ${what} ${error instanceof Error && error.message === 'timeout'
+            ? `did not answer within ${SIGNIN_DB_BACKUP_PORT_TIMEOUT_MS / 1000} seconds`
+            : 'failed'}`);
+    }
+    finally {
+        clearTimeout(timer);
+    }
+}
+const usable = (value) => typeof value === 'string' && value.length > 0 && value.length <= 200;
+/**
+ * The company's own bucket and a storage key that names that bucket alone. A key already in the
+ * database app's env whose token id is recorded is kept. Otherwise any earlier token of this company
+ * (recorded, or minted and never confirmed) is deleted first, so there are never two live tokens, then
+ * one is minted, registered as a secret and recorded as undelivered; the caller writes it to the env and
+ * then records it as delivered. A failed bucket or mint throws before the database is deployed or the
+ * move stops anything. Errors name the label only.
+ */
+export async function ensureSigninDbBackupTarget(port, input) {
+    const { label, recorded } = input;
+    const bucket = await portCall(port.r2Buckets.ensureBucket(input.bucketName), label, `the sign-in backup bucket ${input.bucketName}`);
+    const kept = recorded[SIGNIN_DB_BACKUP_KEY_SCOPE_KEY];
+    if (kept && input.held.keyId && input.held.secret) {
+        input.onSecret?.(input.held.secret);
+        input.onSecret?.(input.held.keyId);
+        return {
+            target: {
+                endpoint: bucket.endpoint,
+                bucket: bucket.bucket,
+                accessKeyId: input.held.keyId,
+                secretAccessKey: input.held.secret,
+            },
+            tokenId: kept,
+            fresh: false,
+        };
+    }
+    for (const [scopeKey, tokenId] of [
+        [
+            SIGNIN_DB_BACKUP_KEY_UNDELIVERED_SCOPE_KEY,
+            recorded[SIGNIN_DB_BACKUP_KEY_UNDELIVERED_SCOPE_KEY],
+        ],
+        [SIGNIN_DB_BACKUP_KEY_SCOPE_KEY, kept],
+    ]) {
+        if (!tokenId)
+            continue;
+        await portCall(port.r2Keys.deleteBucketKey(tokenId), label, `deleting the earlier sign-in backup key ${tokenId}, so no new one was made:`);
+        await input.record(scopeKey, '');
+    }
+    const key = await portCall(port.r2Keys.mintBucketKey(bucket.bucket), label, 'the sign-in backup key mint');
+    if (usable(key?.secretAccessKey))
+        input.onSecret?.(key.secretAccessKey);
+    if (usable(key?.accessKeyId))
+        input.onSecret?.(key.accessKeyId);
+    // A token id we can read is recorded even when the rest of the answer is unusable, so it is deleted next run.
+    if (usable(key?.tokenId))
+        await input.record(SIGNIN_DB_BACKUP_KEY_UNDELIVERED_SCOPE_KEY, key.tokenId);
+    if (!usable(key?.accessKeyId) || !usable(key?.secretAccessKey) || !usable(key?.tokenId)) {
+        throw new Error(`${label}: the sign-in backup key mint answered with an unusable key`);
+    }
+    return {
+        target: {
+            endpoint: bucket.endpoint,
+            bucket: bucket.bucket,
+            accessKeyId: key.accessKeyId,
+            secretAccessKey: key.secretAccessKey,
+        },
+        tokenId: key.tokenId,
+        fresh: true,
+    };
+}
+/** The env of the database app that the backup command reads. The command itself holds no value. */
+export const SIGNIN_DB_BACKUP_ENV = {
+    endpoint: 'BACKUP_S3_ENDPOINT',
+    bucket: 'BACKUP_S3_BUCKET',
+    region: 'BACKUP_S3_REGION',
+    keyId: 'BACKUP_S3_KEY_ID',
+    secret: 'BACKUP_S3_SECRET',
+    prefix: 'BACKUP_S3_PREFIX',
+};
+/** Where in the bucket a company's dumps go: `signin-db/<slug>/signin-db-<UTC time>.dump`. */
+export function signinDbBackupPrefix(slug) {
+    return `signin-db/${slug}`;
+}
+/** The env entries the backup command needs, for the database app. */
+export function signinDbBackupEnvs(target, slug) {
+    return [
+        { key: SIGNIN_DB_BACKUP_ENV.endpoint, value: target.endpoint.replace(/\/+$/, '') },
+        { key: SIGNIN_DB_BACKUP_ENV.bucket, value: target.bucket },
+        { key: SIGNIN_DB_BACKUP_ENV.region, value: target.region ?? 'auto' },
+        { key: SIGNIN_DB_BACKUP_ENV.keyId, value: target.accessKeyId },
+        { key: SIGNIN_DB_BACKUP_ENV.secret, value: target.secretAccessKey },
+        { key: SIGNIN_DB_BACKUP_ENV.prefix, value: signinDbBackupPrefix(slug) },
+    ];
+}
+/**
+ * The command Coolify runs in the container. A custom-format dump goes to a temporary file (an upload
+ * needs a length), is signed with SigV4 and put with curl, and the file is removed on every exit. The
+ * credentials reach curl on its standard input as a config, never in an argument. Every failure ends the
+ * run non-zero, and curl's error text names the HTTP status only. No single quote: Coolify wraps the
+ * command in single quotes.
+ */
+export function signinDbBackupCommand(user, database) {
+    const e = SIGNIN_DB_BACKUP_ENV;
+    return [
+        'set -eu',
+        'f=/tmp/signin-db-$(date -u +%Y%m%dT%H%M%SZ).dump',
+        'trap "rm -f $f" EXIT',
+        `pg_dump -U ${user} -d ${database} -Fc -f "$f"`,
+        `printf "user = \\"%s:%s\\"\\n" "$${e.keyId}" "$${e.secret}" | curl -sS --fail --max-time ${UPLOAD_MAX_SECONDS} --aws-sigv4 "aws:amz:$${e.region}:s3" -K - -T "$f" "$${e.endpoint}/$${e.bucket}/$${e.prefix}/\${f##*/}"`,
+    ].join('; ');
+}
+function rows(body) {
+    if (!Array.isArray(body))
+        throw new Error('scheduled tasks list has an unexpected shape');
+    return body.map((row) => (row && typeof row === 'object' ? row : {}));
+}
+/**
+ * Makes sure the database app has its backup task, and returns its uuid. The task is found by name
+ * before it is created, and updated to the full wanted body every time (Coolify hides the stored
+ * command from a token without the sensitive-read ability, so it cannot be compared). The task is then
+ * read back and must be listed enabled with the wanted frequency and timeout, or this throws: a
+ * schedule that cannot be seen is not a backup. The caller records the uuid. Errors name the label only.
+ */
+export async function ensureSigninDbBackupTask(coolify, appId, label, db) {
+    const base = `/applications/${appId}/scheduled-tasks`;
+    const body = {
+        name: SIGNIN_DB_BACKUP_TASK_NAME,
+        command: signinDbBackupCommand(db.user, db.database),
+        frequency: SIGNIN_DB_BACKUP_FREQUENCY,
+        timeout: SIGNIN_DB_BACKUP_TIMEOUT_SECONDS,
+        enabled: true,
+    };
+    const found = rows(await coolify.raw('GET', base)).find((t) => t.name === SIGNIN_DB_BACKUP_TASK_NAME && typeof t.uuid === 'string');
+    let uuid = found?.uuid;
+    if (found) {
+        await coolify.raw('PATCH', `${base}/${found.uuid}`, body);
+    }
+    else {
+        uuid = (await coolify.raw('POST', base, body))?.uuid;
+    }
+    if (typeof uuid !== 'string' || !uuid) {
+        throw new Error(`${label}: Coolify did not return the sign-in database's backup schedule`);
+    }
+    const listed = rows(await coolify.raw('GET', base)).find((t) => t.uuid === uuid);
+    if (!listed ||
+        listed.enabled !== true ||
+        listed.frequency !== SIGNIN_DB_BACKUP_FREQUENCY ||
+        listed.timeout !== SIGNIN_DB_BACKUP_TIMEOUT_SECONDS) {
+        throw new Error(`${label}: the sign-in database's backup schedule is not listed as enabled in Coolify; nothing was moved`);
+    }
+    return uuid;
+}
+/** A company on the TLS database whose nightly dump is not scheduled yet: the sign-in step plans again to schedule it. */
+export function signinDbBackupMissing(recordedIds) {
+    return (Boolean(recordedIds[SIGNIN_DB_TLS_SCOPE_KEY]) && !recordedIds[SIGNIN_DB_BACKUP_TASK_SCOPE_KEY]);
+}

package/dist/plan/signin-db-move.d.ts

@@ -0,0 +1,68 @@
+import type { ProjectSource, ServiceTarget, StepScope } from './own-services.js';
+import type { PlatformAuthDeps } from './platform-providers.js';
+/**
+ * Moves a company's sign-in data from its Coolify database to the TLS database (#746).
+ *
+ * 1. The new database app is created and healthy (`provisionSigninDb`): ZITADEL is still up.
+ * 2. ZITADEL is stopped, so nothing writes to the old database.
+ * 3. A one-shot job app on the same Docker network empties the new database, runs `pg_dump` from
+ *    the old internal URL into it, and compares the row count of every table in both. It reports
+ *    healthy only after that passes, which is the signal the step waits for.
+ * 4. The CA is mounted on ZITADEL, its DSN is switched, the move is recorded, and ZITADEL starts.
+ *
+ * The old database is never written to and never deleted. After any failure that follows the stop,
+ * the step reads ZITADEL's DSN back from Coolify and acts on what is really set: the old DSN starts
+ * ZITADEL on the old database (the new app and the job app stay for inspection; a rerun copies again),
+ * the new DSN records the move and starts ZITADEL on the new database. A DSN write that timed out
+ * without a verdict, or a DSN that cannot be read, leaves ZITADEL stopped and says so. The move is
+ * recorded before ZITADEL starts on the new database, never after, and a rerun that finds the new DSN
+ * set finishes the move instead of copying, so it never drops a database ZITADEL may have written to.
+ */
+export declare const SIGNIN_DB_MOVE_ENV: {
+    readonly oldUrl: "OLD_DATABASE_URL";
+    readonly newUrl: "NEW_DATABASE_URL";
+    readonly newAdminUrl: "NEW_ADMIN_DATABASE_URL";
+    readonly ca: "NEW_DATABASE_CA_B64";
+};
+/** Recorded id of the copy job app. */
+export declare const SIGNIN_DB_MOVE_APP_SCOPE_KEY = "signin_db_move_app";
+/**
+ * The job's script. Every line is plain shell; the Dockerfile carries it base64-encoded, so the SQL
+ * may hold quotes. It prints table counts and its own messages, never a URL.
+ */
+export declare function signinDbMoveScript(): string[];
+/** The copy job: the same pinned image as the database, so `pg_dump` matches the server's major version. */
+export declare function signinDbMoveDockerfile(): string;
+/** A URL the job connects to the new database with; the host name is checked against the certificate. */
+export declare function signinDbMoveNewUrl(host: string, password: string, database: string): string;
+export interface MoveRuntime {
+    sleep?: (ms: number) => Promise<void>;
+    now?: () => number;
+    onSecret?: (value: string) => void;
+    /** How long the copy job may run (default 30 minutes). */
+    signinDbMoveTimeoutMs?: number;
+    /** Pause between polls of the job (default 10 seconds). */
+    signinDbMovePollMs?: number;
+    /** How long ZITADEL may take to report stopped before the copy (default 2 minutes). */
+    signinDbStopTimeoutMs?: number;
+    /** How long ZITADEL may take to report healthy on the new database (default 5 minutes). */
+    readyTimeoutMs?: number;
+    /** How long the new database may take to report healthy (default 5 minutes). */
+    signinDbTimeoutMs?: number;
+}
+export interface MoveInput {
+    project: ProjectSource;
+    label: string;
+}
+/**
+ * The step. Returns without doing anything once the move is recorded. See the file comment for the
+ * order and for what a failure at each stage leaves behind.
+ */
+export declare function moveSigninDatabase(scope: StepScope, deps: Pick<PlatformAuthDeps, 'coolify' | 'heldKeys' | 'signinDbBackup'> & {
+    runtime?: MoveRuntime;
+    /** Reads the old database without the resource-limits patch the limited port adds: the old database is only read. */
+    readOldDatabase(uuid: string): Promise<{
+        internal_db_url?: string;
+        status?: string;
+    }>;
+}, target: ServiceTarget, input: MoveInput): Promise<void>;

package/dist/plan/signin-db-move.js

@@ -0,0 +1,338 @@
+import { pinnedRef } from '../bootstrap/versions.js';
+import { PLATFORM_AUTH_DB_SCOPE_KEY, PLATFORM_AUTH_ZITADEL_SCOPE_KEY, findOrCreatePlatformApp, } from './platform-providers.js';
+import { SIGNIN_DB_TLS_SCOPE_KEY, SIGNIN_DB_ZITADEL_PENDING_SCOPE_KEY } from './signin-db-tls.js';
+import { SIGNIN_DB_NAME, SIGNIN_DB_PORT, SIGNIN_DB_USER, boundedCall, dockerfileAppCreator, ensureSigninDbCaFile, provisionSigninDb, waitHealthy, withCallTimeout, } from './signin-db.js';
+/**
+ * Moves a company's sign-in data from its Coolify database to the TLS database (#746).
+ *
+ * 1. The new database app is created and healthy (`provisionSigninDb`): ZITADEL is still up.
+ * 2. ZITADEL is stopped, so nothing writes to the old database.
+ * 3. A one-shot job app on the same Docker network empties the new database, runs `pg_dump` from
+ *    the old internal URL into it, and compares the row count of every table in both. It reports
+ *    healthy only after that passes, which is the signal the step waits for.
+ * 4. The CA is mounted on ZITADEL, its DSN is switched, the move is recorded, and ZITADEL starts.
+ *
+ * The old database is never written to and never deleted. After any failure that follows the stop,
+ * the step reads ZITADEL's DSN back from Coolify and acts on what is really set: the old DSN starts
+ * ZITADEL on the old database (the new app and the job app stay for inspection; a rerun copies again),
+ * the new DSN records the move and starts ZITADEL on the new database. A DSN write that timed out
+ * without a verdict, or a DSN that cannot be read, leaves ZITADEL stopped and says so. The move is
+ * recorded before ZITADEL starts on the new database, never after, and a rerun that finds the new DSN
+ * set finishes the move instead of copying, so it never drops a database ZITADEL may have written to.
+ */
+export const SIGNIN_DB_MOVE_ENV = {
+    oldUrl: 'OLD_DATABASE_URL',
+    newUrl: 'NEW_DATABASE_URL',
+    newAdminUrl: 'NEW_ADMIN_DATABASE_URL',
+    ca: 'NEW_DATABASE_CA_B64',
+};
+/** Where the job writes the CA it is given in env. */
+const JOB_CA_PATH = '/tmp/ca.crt';
+const JOB_DONE_FILE = '/tmp/done';
+/** Left by a run that did not finish: a restart of the same container then stops at once instead of copying again. */
+const JOB_FAILED_FILE = '/tmp/failed';
+const JOB_DUMP_FILE = '/tmp/dump.sql';
+/** Recorded id of the copy job app. */
+export const SIGNIN_DB_MOVE_APP_SCOPE_KEY = 'signin_db_move_app';
+/**
+ * The job's script. Every line is plain shell; the Dockerfile carries it base64-encoded, so the SQL
+ * may hold quotes. It prints table counts and its own messages, never a URL.
+ */
+export function signinDbMoveScript() {
+    const old = `"$${SIGNIN_DB_MOVE_ENV.oldUrl}"`;
+    const next = `"$${SIGNIN_DB_MOVE_ENV.newUrl}"`;
+    const admin = `"$${SIGNIN_DB_MOVE_ENV.newAdminUrl}"`;
+    const tables = "select quote_ident(schemaname) || '.' || quote_ident(tablename) from pg_tables where schemaname not in ('pg_catalog', 'information_schema') order by 1";
+    return [
+        '#!/bin/sh',
+        'set -eu',
+        // The dump holds the whole sign-in database: it is removed on every exit. A run that ends without
+        // `done` leaves `failed`, because Coolify restarts the container and a restart must not copy again.
+        `trap 'rm -f ${JOB_DUMP_FILE}; [ -f ${JOB_DONE_FILE} ] || touch ${JOB_FAILED_FILE}' EXIT`,
+        "trap 'exit 1' HUP INT TERM",
+        `[ ! -f ${JOB_FAILED_FILE} ] || exit 1`,
+        `rm -f ${JOB_DONE_FILE}`,
+        `printf %s "$${SIGNIN_DB_MOVE_ENV.ca}" | base64 -d > ${JOB_CA_PATH}`,
+        'fail() { echo "signin-db-move: $1"; exit 1; }',
+        `major=$(psql ${old} -Atc "show server_version_num" | cut -c1-2)`,
+        '[ "$major" = "16" ] || fail "the old database is not Postgres 16"',
+        `psql ${admin} -v ON_ERROR_STOP=1 -q -c "drop database if exists ${SIGNIN_DB_NAME} with (force)" -c "create database ${SIGNIN_DB_NAME}" || fail "the new database could not be emptied"`,
+        `pg_dump ${old} > ${JOB_DUMP_FILE} || fail "pg_dump failed"`,
+        `psql ${next} -v ON_ERROR_STOP=1 -q -f ${JOB_DUMP_FILE} > /dev/null || fail "the restore failed"`,
+        `rm -f ${JOB_DUMP_FILE}`,
+        `old_tables=$(psql ${old} -Atc "${tables}")`,
+        `new_tables=$(psql ${next} -Atc "${tables}")`,
+        '[ "$old_tables" = "$new_tables" ] || fail "the two databases hold different tables"',
+        'checked=0',
+        'for t in $old_tables; do',
+        `  a=$(psql ${old} -Atc "select count(*) from $t")`,
+        `  b=$(psql ${next} -Atc "select count(*) from $t")`,
+        '  [ "$a" = "$b" ] || fail "row count differs in $t: old $a, new $b"',
+        '  checked=$((checked + 1))',
+        'done',
+        '[ "$checked" -gt 0 ] || fail "the old database has no tables"',
+        'echo "signin-db-move: $checked tables copied, row counts equal"',
+        `touch ${JOB_DONE_FILE}`,
+        'exec sleep infinity',
+    ];
+}
+/** The copy job: the same pinned image as the database, so `pg_dump` matches the server's major version. */
+export function signinDbMoveDockerfile() {
+    const encoded = Buffer.from(`${signinDbMoveScript().join('\n')}\n`, 'utf8').toString('base64');
+    return [
+        `FROM postgres:${pinnedRef('postgres')}`,
+        `RUN echo '${encoded}' | base64 -d > /usr/local/bin/signin-db-move && chmod 0755 /usr/local/bin/signin-db-move`,
+        'ENTRYPOINT ["/usr/local/bin/signin-db-move"]',
+        `HEALTHCHECK --interval=10s --timeout=5s --start-period=10s --retries=1 CMD ["test", "-f", "${JOB_DONE_FILE}"]`,
+    ].join('\n');
+}
+/** A URL the job connects to the new database with; the host name is checked against the certificate. */
+export function signinDbMoveNewUrl(host, password, database) {
+    return `postgres://${SIGNIN_DB_USER}:${password}@${host}:${SIGNIN_DB_PORT}/${database}?sslmode=verify-full&sslrootcert=${JOB_CA_PATH}`;
+}
+/** A failure the step words itself. Anything else that goes wrong is reported without its cause, which could carry a URL. */
+class MoveFailure extends Error {
+}
+const defaultSleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
+/** Polls of an `exited` job with no `running` seen before it counts as a stop (an `exited` read can be the container before the deploy). */
+const JOB_EXITED_POLLS = 6;
+/**
+ * Waits for the copy job to report healthy, which it does only after the counts match. The job never
+ * exits on success, so any exit is final: `restarting` at once, `exited` once the job was seen running
+ * or stayed exited for {@link JOB_EXITED_POLLS} polls. It has failed too at the deadline. A healthy
+ * status read before any other could be the container before the stop, so it counts only after a
+ * different status was seen.
+ */
+async function waitForJob(coolify, appId, runtime) {
+    const now = runtime.now ?? Date.now;
+    const sleep = runtime.sleep ?? defaultSleep;
+    const timeout = runtime.signinDbMoveTimeoutMs ?? 1_800_000;
+    const interval = runtime.signinDbMovePollMs ?? 10_000;
+    const deadline = now() + timeout;
+    let left = false;
+    let running = false;
+    let exitedPolls = 0;
+    for (;;) {
+        const { status } = await coolify.getApplication(appId);
+        if (status === 'running:healthy' && left)
+            return;
+        if (status !== 'running:healthy')
+            left = true;
+        if (status?.startsWith('running'))
+            running = true;
+        exitedPolls = status?.startsWith('exited') ? exitedPolls + 1 : 0;
+        if (status?.startsWith('restarting') ||
+            (status?.startsWith('exited') && (running || exitedPolls >= JOB_EXITED_POLLS))) {
+            throw new MoveFailure('the copy job stopped before the row counts matched');
+        }
+        if (now() >= deadline) {
+            throw new MoveFailure(`the copy job did not finish within ${Math.round(timeout / 1000)}s (last status: ${status ?? 'unknown'})`);
+        }
+        await sleep(interval);
+    }
+}
+/** Coolify queues a stop. Waits until it reports the app stopped; `pg_dump` must not run while ZITADEL can still write. */
+async function waitStopped(coolify, appId, runtime) {
+    const now = runtime.now ?? Date.now;
+    const sleep = runtime.sleep ?? defaultSleep;
+    const timeout = runtime.signinDbStopTimeoutMs ?? 120_000;
+    const interval = runtime.signinDbMovePollMs ?? 10_000;
+    const deadline = now() + timeout;
+    for (;;) {
+        const { status } = await coolify.getApplication(appId);
+        if (status?.startsWith('exited') || status?.startsWith('stopped'))
+            return;
+        if (now() >= deadline) {
+            throw new MoveFailure(`ZITADEL did not report stopped within ${Math.round(timeout / 1000)}s (last status: ${status ?? 'unknown'}), so nothing was copied`);
+        }
+        await sleep(interval);
+    }
+}
+/** Runs `work`; a failure comes back as a note for the error message, never thrown. */
+async function attempt(work, failed) {
+    try {
+        await work();
+        return '';
+    }
+    catch (error) {
+        return error instanceof MoveFailure ? ` (${failed}: ${error.message})` : ` (${failed})`;
+    }
+}
+/**
+ * The step. Returns without doing anything once the move is recorded. See the file comment for the
+ * order and for what a failure at each stage leaves behind.
+ */
+export async function moveSigninDatabase(scope, deps, target, input) {
+    const coolify = withCallTimeout(deps.coolify);
+    const { label } = input;
+    const runtime = deps.runtime ?? {};
+    const recorded = scope.recordedIds();
+    if (recorded[SIGNIN_DB_TLS_SCOPE_KEY])
+        return;
+    const oldDbId = recorded[PLATFORM_AUTH_DB_SCOPE_KEY];
+    const zitadelId = recorded[PLATFORM_AUTH_ZITADEL_SCOPE_KEY];
+    if (!oldDbId || !zitadelId) {
+        throw new Error(`${label}: the sign-in database cannot be moved: the company's sign-in is not applied yet`);
+    }
+    const record = async (key, id) => {
+        scope.recordScopeId(key, id);
+        await scope.persist();
+    };
+    const old = await boundedCall(deps.readOldDatabase(oldDbId), 'Coolify getDatabase');
+    if (!old.internal_db_url || !old.status?.startsWith('running')) {
+        throw new Error(`${label}: the sign-in database cannot be moved: the old database is not running`);
+    }
+    runtime.onSecret?.(old.internal_db_url);
+    // The password alone too: a message that quotes only that part must be scrubbed as well.
+    if (URL.canParse(old.internal_db_url)) {
+        const password = decodeURIComponent(new URL(old.internal_db_url).password);
+        if (password)
+            runtime.onSecret?.(password);
+    }
+    const projectUuid = await input.project.ensure(coolify, scope);
+    const { environment } = await coolify.findOrCreateEnvironment(projectUuid, 'production');
+    const creator = (name, description) => dockerfileAppCreator(coolify, { projectUuid, environment, name, description });
+    // (b) The new database. ZITADEL is still up on the old one.
+    const db = await provisionSigninDb({
+        coolify,
+        keys: deps.heldKeys,
+        runtime,
+        name: target.names.signinDb,
+        internalHost: target.internal.signinDb,
+        volumeName: `${target.names.signinDb}-data`,
+        serverCertRef: target.refs.signinDbServer,
+        slug: target.id,
+        backup: deps.signinDbBackup,
+        backupBucket: target.names.signinBackupBucket,
+        recorded,
+        label,
+        environment,
+        record,
+        create: creator(target.names.signinDb, `${target.id}'s sign-in database with TLS (#746)`),
+    });
+    const newUrl = signinDbMoveNewUrl(target.internal.signinDb, db.password, SIGNIN_DB_NAME);
+    const adminUrl = signinDbMoveNewUrl(target.internal.signinDb, db.password, 'postgres');
+    runtime.onSecret?.(newUrl);
+    runtime.onSecret?.(adminUrl);
+    const dsnKey = 'ZITADEL_DATABASE_POSTGRES_DSN';
+    /** Which database ZITADEL's DSN names in Coolify right now. */
+    const liveDatabase = async () => {
+        try {
+            const dsn = (await coolify.listEnvs(zitadelId)).find((e) => e.key === dsnKey && !e.is_preview)?.value;
+            if (dsn === db.dsn)
+                return 'new';
+            return dsn === old.internal_db_url ? 'old' : 'unknown';
+        }
+        catch {
+            // The cause may carry a URL; the caller words the state.
+            return 'unknown';
+        }
+    };
+    /**
+     * ZITADEL's DSN names the new database. The move is recorded, with the flag that ZITADEL is not yet
+     * seen healthy there (it makes the planner plan the sign-in step), in one write before ZITADEL
+     * starts. Every failure says the exact state.
+     */
+    const startOnNew = async () => {
+        scope.recordScopeId(SIGNIN_DB_TLS_SCOPE_KEY, db.appId);
+        scope.recordScopeId(SIGNIN_DB_ZITADEL_PENDING_SCOPE_KEY, '1');
+        try {
+            await scope.persist();
+        }
+        catch {
+            throw new MoveFailure('ZITADEL is stopped and its DSN names the new database; the move could not be recorded, so ZITADEL was not started on it; apply again, which records the move and starts it');
+        }
+        try {
+            await coolify.deploy(zitadelId);
+        }
+        catch {
+            throw new MoveFailure('the move is recorded and ZITADEL uses the new database, but it could not be started; start it in Coolify (the next apply starts it too)');
+        }
+        try {
+            await waitHealthy(coolify, zitadelId, { ...runtime, signinDbTimeoutMs: runtime.readyTimeoutMs ?? 300_000 }, label, 'ZITADEL');
+        }
+        catch (error) {
+            // waitHealthy words its own message (the app, the status); nothing else is repeated.
+            const why = error instanceof Error && error.message.includes('did not report healthy')
+                ? ''
+                : ' (a Coolify call failed)';
+            throw new MoveFailure(`ZITADEL did not report healthy on the new sign-in database${why}; the move is recorded, the old database is untouched, check ZITADEL's logs and apply again`);
+        }
+        await record(SIGNIN_DB_ZITADEL_PENDING_SCOPE_KEY, '');
+    };
+    // An earlier try that got as far as the DSN switch: ZITADEL may have written to the new database,
+    // so it is never emptied again. The move is finished instead.
+    const finished = (error) => {
+        throw new Error(`${label}: ${error instanceof MoveFailure ? error.message : 'the move could not be finished'}`);
+    };
+    const before = await liveDatabase();
+    if (before === 'new') {
+        await startOnNew().catch(finished);
+        return;
+    }
+    if (before === 'unknown') {
+        throw new Error(`${label}: the sign-in database cannot be moved: ZITADEL's DSN could not be read or names neither database, so nothing is copied; ZITADEL may still be stopped from an earlier attempt and this apply does not start it: read the DSN in Coolify, set it to one of the two databases, then apply again`);
+    }
+    let dsnWriteStarted = false;
+    let jobAppId;
+    try {
+        // (c) Nothing writes to the old database from here, once Coolify reports ZITADEL stopped.
+        await coolify.raw('POST', `/applications/${zitadelId}/stop`);
+        await waitStopped(coolify, zitadelId, runtime);
+        // (d) The copy job.
+        const job = await findOrCreatePlatformApp(coolify, target.names.signinDbMove, environment, () => creator(target.names.signinDbMove, `${target.id}'s sign-in database copy job (#746)`)(signinDbMoveDockerfile()));
+        jobAppId = job.appId;
+        await record(SIGNIN_DB_MOVE_APP_SCOPE_KEY, job.appId);
+        await coolify.updateApplication(job.appId, { health_check_enabled: false });
+        await coolify.bulkUpsertEnvs(job.appId, [
+            { key: SIGNIN_DB_MOVE_ENV.oldUrl, value: old.internal_db_url },
+            { key: SIGNIN_DB_MOVE_ENV.newUrl, value: newUrl },
+            { key: SIGNIN_DB_MOVE_ENV.newAdminUrl, value: adminUrl },
+            { key: SIGNIN_DB_MOVE_ENV.ca, value: Buffer.from(db.ca, 'utf8').toString('base64') },
+        ]);
+        // A job left running by a failed run would still read as healthy or as running.
+        if (job.running)
+            await coolify.raw('POST', `/applications/${job.appId}/stop`);
+        await coolify.deploy(job.appId, true);
+        await waitForJob(coolify, job.appId, runtime);
+        await coolify.raw('POST', `/applications/${job.appId}/stop`);
+        // (e) ZITADEL's side of the switch. The write may land even when the call does not answer.
+        await ensureSigninDbCaFile(coolify, zitadelId, db.ca, label);
+        dsnWriteStarted = true;
+        await coolify.bulkUpsertEnvs(zitadelId, [{ key: dsnKey, value: db.dsn }]);
+    }
+    catch (error) {
+        const reason = error instanceof MoveFailure
+            ? error.message
+            : 'a Coolify call failed (its message is not repeated)';
+        const timedOut = error instanceof Error && error.message.includes('did not answer within');
+        let state;
+        let tail = 'the old database is untouched and the new one is left for inspection';
+        const live = await liveDatabase();
+        if (live === 'new') {
+            tail = 'the old database is untouched';
+            try {
+                await startOnNew();
+                state = 'ZITADEL was started on the NEW database and reports healthy; the move is recorded';
+            }
+            catch (startError) {
+                state = `ZITADEL uses the NEW database, but ${startError instanceof MoveFailure ? startError.message : 'it could not be started'}`;
+            }
+        }
+        else if (live === 'old' && !(dsnWriteStarted && timedOut)) {
+            state = `ZITADEL was started again on the old database${await attempt(() => coolify.deploy(zitadelId), 'ZITADEL could not be restarted, start it by hand')}`;
+        }
+        else {
+            state =
+                live === 'old'
+                    ? 'ZITADEL is left stopped: its DSN still names the old database, but the write of the new one did not answer and may still land; do not start ZITADEL by hand, because writes on the old database would be lost if the new DSN lands later; wait until its DSN reads the same twice a minute apart in Coolify, then apply again'
+                    : 'ZITADEL is left stopped: its DSN could not be read or names neither database; do not start ZITADEL by hand; set its DSN to the old database in Coolify, then apply again';
+        }
+        const stoppedJob = jobAppId
+            ? await attempt(() => coolify.raw('POST', `/applications/${jobAppId}/stop`), 'the copy job could not be stopped')
+            : '';
+        throw new Error(`${label}: moving the sign-in database failed: ${reason}. ${state}${stoppedJob}; ${tail}`);
+    }
+    // Recorded before ZITADEL starts on the new database, so a rerun never copies over what it writes there.
+    await startOnNew().catch(finished);
+}