@wtfalch/foundry 0.49.0 → 0.50.1

package/dist/plan/stage-copy-port.js

@@ -0,0 +1,208 @@
+import { execFile } from 'node:child_process';
+import { mkdtemp, rm, writeFile } from 'node:fs/promises';
+import { tmpdir } from 'node:os';
+import { join } from 'node:path';
+import { ensureRuntimeRole } from '@wtfalch/db/runtime-role';
+import { runtimeDatabaseUrl } from './blue-green-apply.js';
+import { withTimeout } from './call-timeout.js';
+import { tableExcluded } from './stage-copy.js';
+/** Thrown before anything is run or fetched. Fixed text: never the URL it refused. */
+export class StageRestoreRefusedError extends Error {
+    constructor(reason) {
+        super(`stage restore refused: ${reason}`);
+        this.name = 'StageRestoreRefusedError';
+    }
+}
+/** Thrown when a step of the restore failed. Fixed text only: the cause can echo a connection URL. */
+export class StageRestoreError extends Error {
+    constructor(step) {
+        super(`stage restore: ${step} failed`);
+        this.name = 'StageRestoreError';
+    }
+}
+/** The default {@link Runner}: `execFile`, stderr dropped, no shell. */
+export const execRunner = (command, args, { env, timeoutMs, signal }) => new Promise((resolve, reject) => {
+    execFile(command, [...args], {
+        env: { ...env },
+        timeout: timeoutMs,
+        maxBuffer: 64 * 1024 * 1024,
+        encoding: 'utf8',
+        ...(signal ? { signal } : {}),
+    }, (error, stdout) => {
+        if (!error)
+            return resolve({ exitCode: 0, stdout });
+        const code = error.code;
+        if (typeof code === 'number')
+            return resolve({ exitCode: code, stdout });
+        reject(new StageRestoreError('the program'));
+    });
+});
+function parseTarget(url) {
+    let parsed;
+    try {
+        parsed = new URL(url);
+    }
+    catch {
+        throw new StageRestoreRefusedError('the database URL is not a URL');
+    }
+    if (parsed.protocol !== 'postgres:' && parsed.protocol !== 'postgresql:') {
+        throw new StageRestoreRefusedError('the database URL is not a postgres URL');
+    }
+    const database = decodeURIComponent(parsed.pathname.replace(/^\//, ''));
+    if (!parsed.hostname || !database) {
+        throw new StageRestoreRefusedError('the database URL names no host or database');
+    }
+    return {
+        host: parsed.hostname.toLowerCase(),
+        port: parsed.port || '5432',
+        user: decodeURIComponent(parsed.username),
+        password: decodeURIComponent(parsed.password),
+        database,
+    };
+}
+const sameDatabase = (a, b) => a.host === b.host && a.port === b.port && a.database === b.database;
+const IDENTIFIER = /^[a-z_][a-z0-9_]*$/;
+const STAGE_DATABASE = /_stage_db$/;
+const pgEnv = (t) => ({
+    PATH: process.env.PATH ?? '',
+    PGHOST: t.host,
+    PGPORT: t.port,
+    PGUSER: t.user,
+    PGPASSWORD: t.password,
+    PGDATABASE: t.database,
+});
+/** `4213; 0 16405 TABLE DATA public sessions owner` */
+const TABLE_DATA = /^\d+;\s+\d+\s+\d+\s+TABLE DATA\s+\S+\s+(\S+)\s+\S+\s*$/;
+/** Any table data entry, parsed or not. */
+const TABLE_DATA_ENTRY = /^\d+;.*\bTABLE DATA\b/;
+/** The table of contents of a dump without the data of the excluded tables. */
+export function filterToc(toc, patterns) {
+    let excluded = 0;
+    const kept = toc.split('\n').filter((line) => {
+        const table = TABLE_DATA.exec(line)?.[1];
+        // A name the parser cannot read (quoted, with a space) could be an excluded table: refuse the restore.
+        if ((table === undefined && TABLE_DATA_ENTRY.test(line)) || table?.includes('"')) {
+            throw new StageRestoreRefusedError('the dump list has a table entry that cannot be read');
+        }
+        if (table !== undefined && tableExcluded(table, patterns)) {
+            excluded += 1;
+            return false;
+        }
+        return true;
+    });
+    return { list: kept.join('\n'), excluded };
+}
+/**
+ * The concrete {@link StageCopyPort}. It restores a dump into a stage database and sets up that database's runtime
+ * role, and refuses any target that is not a stage database or that is production's. Connection settings reach
+ * the programs as `PG*` env, never as arguments. Every outside call has a timeout, and an error carries fixed text
+ * only.
+ */
+export function createStageCopyPort(deps) {
+    const run = deps.run ?? execRunner;
+    const t = deps.timeouts ?? {};
+    const fetchMs = t.fetchMs ?? 600_000;
+    const listMs = t.listMs ?? 120_000;
+    const restoreMs = t.restoreMs ?? 1_800_000;
+    const roleMs = t.roleMs ?? 120_000;
+    for (const [name, value] of Object.entries({ fetchMs, listMs, restoreMs, roleMs })) {
+        if (!(value > 0))
+            throw new TypeError(`stage copy port: ${name} must be positive`);
+    }
+    /** Parses the target and refuses it unless it is a stage database that production does not answer at. */
+    async function checkTarget(url, role) {
+        const target = parseTarget(url);
+        if (!IDENTIFIER.test(role))
+            throw new StageRestoreRefusedError('the role name is not valid');
+        if (!STAGE_DATABASE.test(target.database)) {
+            throw new StageRestoreRefusedError('the target is not a stage database');
+        }
+        // The pg client reads host, port and options from the query string: only `sslmode` is allowed.
+        for (const key of new URL(url).searchParams.keys()) {
+            if (key !== 'sslmode') {
+                throw new StageRestoreRefusedError('the database URL has query parameters');
+            }
+        }
+        const production = await withTimeout('stage restore: production URLs', roleMs, deps.productionUrls);
+        if (production.length === 0) {
+            throw new StageRestoreRefusedError('production database is not known');
+        }
+        for (const url of production) {
+            if (sameDatabase(target, parseTarget(url))) {
+                throw new StageRestoreRefusedError('the target is the production database');
+            }
+        }
+        return target;
+    }
+    return {
+        async runRestore({ location, targetDatabaseUrl, ownerRole, excludeTableData, signal }) {
+            const target = await checkTarget(targetDatabaseUrl, ownerRole);
+            if (target.database !== ownerRole) {
+                throw new StageRestoreRefusedError('the owner role is not the stage database owner');
+            }
+            const fetching = deps.fetchDump(location, signal);
+            let dump;
+            try {
+                dump = await withTimeout('stage restore: fetch the dump', fetchMs, () => fetching);
+            }
+            catch (error) {
+                // The fetch may still finish later; its file goes as soon as it does.
+                fetching.then((late) => late.remove().catch(() => undefined), () => undefined);
+                throw error;
+            }
+            let dir;
+            try {
+                const env = pgEnv(target);
+                const toc = await withTimeout('stage restore: list the dump', listMs, () => run('pg_restore', ['--list', dump.path], { env, timeoutMs: listMs, signal }));
+                if (toc.exitCode !== 0)
+                    return { exitCode: toc.exitCode };
+                dir = await mkdtemp(join(tmpdir(), 'stage-restore-'));
+                const listFile = join(dir, 'toc.list');
+                await writeFile(listFile, filterToc(toc.stdout, excludeTableData).list, 'utf8');
+                const restored = await withTimeout('stage restore: pg_restore', restoreMs, () => run('pg_restore', [
+                    '--no-owner',
+                    '--no-privileges',
+                    '--exit-on-error',
+                    `--role=${ownerRole}`,
+                    '--use-list',
+                    listFile,
+                    '--dbname',
+                    target.database,
+                    dump.path,
+                ], { env, timeoutMs: restoreMs, signal }));
+                return { exitCode: restored.exitCode };
+            }
+            catch (error) {
+                if (error instanceof StageRestoreRefusedError)
+                    throw error;
+                console.error(`stage restore: failed (${error instanceof Error ? error.name : 'unknown'})`);
+                throw new StageRestoreError('the restore');
+            }
+            finally {
+                await dump.remove().catch(() => {
+                    console.error('stage restore: removing the dump file failed');
+                });
+                if (dir)
+                    await rm(dir, { recursive: true, force: true }).catch(() => undefined);
+            }
+        },
+        async ensureRuntimeRole({ targetDatabaseUrl, runtimeRole, runtimePassword }) {
+            const target = await checkTarget(targetDatabaseUrl, runtimeRole);
+            const runtimeUrl = runtimeDatabaseUrl(targetDatabaseUrl, runtimeRole, runtimePassword);
+            if (!runtimeUrl)
+                throw new StageRestoreRefusedError('the database URL is not a postgres URL');
+            if (!runtimeRole.endsWith('_rt') || runtimeRole === target.user) {
+                throw new StageRestoreRefusedError('the runtime role is not a separate runtime role');
+            }
+            try {
+                const ensure = deps.ensureRole ?? ((options) => ensureRuntimeRole(options));
+                await withTimeout('stage restore: runtime role', roleMs, () => ensure({ ownerUrl: targetDatabaseUrl, runtimeUrl, log: () => undefined }));
+                return { exitCode: 0 };
+            }
+            catch (error) {
+                console.error(`stage restore: runtime role failed (${error instanceof Error ? error.name : 'unknown'})`);
+                return { exitCode: 1 };
+            }
+        },
+    };
+}

package/dist/plan/stage-copy.d.ts

@@ -0,0 +1,140 @@
+import { type BlueGreenSubject } from './blue-green.js';
+import { type DatabaseDump } from './database-dump.js';
+import type { CoolifyPort } from './executor.js';
+/**
+ * Table names whose rows the exact copy leaves out (SQL structure is still restored). `*` matches any run of
+ * characters. Sessions and tokens are dropped so a copied sign-in or API token is useless on stage; the keystore
+ * and held-key tables hold ciphertext that stage must never be able to open (stage gets no production key).
+ */
+export declare const STAGE_COPY_EXCLUDED_TABLES: readonly string[];
+/** Whether `table` is on {@link STAGE_COPY_EXCLUDED_TABLES}. Case-insensitive, like Postgres folds names. */
+export declare function tableExcluded(table: string, patterns?: readonly string[]): boolean;
+export type StageCopyRefusalReason = 'guards' | 'sink' | 'stage-sink-env' | 'stage-isolation' | 'no-production-database' | 'no-stage-database' | 'no-stage-app' | 'no-coolify-project' | 'no-kept-database';
+/** Thrown before anything is touched. Fixed text and names only. */
+export declare class StageCopyRefusedError extends Error {
+    readonly reason: StageCopyRefusalReason;
+    readonly missing: readonly string[];
+    constructor(reason: StageCopyRefusalReason, missing?: readonly string[]);
+}
+export interface StageCopyStepRecord {
+    step: string;
+    outcome: 'ok' | 'failed';
+    /** Fixed text only. */
+    detail?: string;
+}
+/** What a run did. Fixed text, ids and times only: never a URL, a password or a Coolify message. */
+export interface StageCopyRunRecord {
+    requestedBy: string;
+    at: string;
+    /** The Coolify database stage runs on when the call ended. */
+    liveDatabaseId: string;
+    dumpExecutionId?: string;
+    steps: readonly StageCopyStepRecord[];
+}
+/** Thrown once a run started; carries what happened so far. */
+export declare class StageCopyFailedError extends Error {
+    readonly record: StageCopyRunRecord;
+    constructor(message: string, record: StageCopyRunRecord);
+}
+/** The restore and the runtime role, run by the host (`stage-copy-port.ts` is the concrete one). */
+export interface StageCopyPort {
+    /**
+     * Restores the dump at `location` into `targetDatabaseUrl` and nothing else, leaving out the data of every
+     * table matching `excludeTableData` (see {@link tableExcluded}).
+     */
+    runRestore(input: {
+        location: DatabaseDump['location'];
+        targetDatabaseUrl: string;
+        ownerRole: string;
+        excludeTableData: readonly string[];
+        /** Aborted when the caller gives up: the restore must stop writing and remove the dump file. */
+        signal?: AbortSignal;
+    }): Promise<{
+        exitCode: number;
+    }>;
+    /** Creates or syncs `runtimeRole` with `runtimePassword` on the database `targetDatabaseUrl` names. */
+    ensureRuntimeRole(input: {
+        targetDatabaseUrl: string;
+        runtimeRole: string;
+        runtimePassword: string;
+    }): Promise<{
+        exitCode: number;
+    }>;
+}
+export type StageCopyCoolify = Pick<CoolifyPort, 'listDatabases' | 'getDatabase' | 'createPostgresDatabase' | 'deleteDatabase' | 'getApplication' | 'listEnvs' | 'deleteEnv' | 'deployAndWait' | 'bulkUpsertEnvs' | 'findOrCreateEnvironment' | 'listDatabaseBackups' | 'raw'> & Partial<Pick<CoolifyPort, 'listDatabaseBackupExecutions'>>;
+/** Timeouts of every outside call; a run has no call without one. */
+export interface StageCopyTimeouts {
+    /** One Coolify call. Default 30 s. */
+    callMs?: number;
+    /** The dump of production. Default: `dumpNow`'s (10 minutes). */
+    dumpMs?: number;
+    /**
+     * The whole `runRestore` call: the port's fetch (10 minutes), table list (2) and `pg_restore` (30), plus a
+     * margin. Default 44 minutes.
+     */
+    restoreMs?: number;
+    /** Creating or syncing the runtime role. Default 2 minutes. */
+    roleMs?: number;
+    /** Waiting for a new database to start. Default 3 minutes. */
+    databaseStartMs?: number;
+    /** The stage app's new deployment. Default 16 minutes. */
+    deployMs?: number;
+    /** Waiting for the stage app to be healthy after its deployment, and the poll between reads. Default 2 minutes, 2 s. */
+    appRunningMs?: number;
+    appPollMs?: number;
+}
+/** The production database to dump, resolved by the host (it differs by app). `null`: none is recorded. */
+export interface StageCopyProduction {
+    /** The database's name in the backup keys, e.g. `app.shop`. Goes into dump errors. */
+    databaseKey: string;
+    databaseUuid: string;
+    /** The uuid of the schedule that saves it to the backup bucket. */
+    backupUuid: string;
+    storageUuid: string;
+}
+interface StageCopyBase {
+    subject: Pick<BlueGreenSubject, 'id' | 'base' | 'database'>;
+    recorded: () => Readonly<Record<string, string>>;
+    record: (key: string, value: string) => void;
+    persist: () => Promise<void>;
+    coolify: StageCopyCoolify;
+    /** Registers a value for redaction before it is written anywhere. */
+    secret?: (value: string) => void;
+    /** Who asked: a person or operator id. Goes into the run record. */
+    requestedBy: string;
+    now?: () => number;
+    timeouts?: StageCopyTimeouts;
+}
+export interface StageCopyInput extends StageCopyBase {
+    /** The company whose sink the stage mail env must name. */
+    org: {
+        slug: string;
+    };
+    production: StageCopyProduction | null;
+    restore: StageCopyPort;
+    projectUuid: string | undefined;
+    serverUuid: string | undefined;
+}
+export interface UndoStageCopyInput extends StageCopyBase {
+    restore: Pick<StageCopyPort, 'ensureRuntimeRole'>;
+}
+/**
+ * Copies production's database into stage, exact except for {@link STAGE_COPY_EXCLUDED_TABLES}, on request. It
+ * refuses until the stage guards and the sink are in place, dumps production through Coolify (it never holds a
+ * production URL), restores into a new database beside stage's and points stage at it. Stage's database is
+ * recorded only once stage runs on the new one.
+ *
+ * What a failure leaves behind, by point: refusal or dump failure: nothing changed. New database made, restore
+ * or role failed: the new database is deleted, stage untouched. Switch failed: stage is pointed back at its own
+ * database and the new one deleted; if putting it back also failed the run says so (`recovered: false`) and the
+ * new database is left for the next run's sweep. Recording: the three records are written in memory and
+ * persisted once, so a crash leaves either all or none.
+ */
+export declare function stageCopy(input: StageCopyInput): Promise<StageCopyRunRecord>;
+/**
+ * Puts the database stage ran on before the last copy back, and deletes the one the copy made. It mints a new
+ * runtime password for the kept database (the old one is gone from stage's env). A failure leaves stage on the
+ * database it ran on, switched back the same way a failed copy does, and records nothing.
+ */
+export declare function undoStageCopy(input: UndoStageCopyInput): Promise<StageCopyRunRecord>;
+export {};