@rebasepro/server-postgres 0.0.1-canary.4829d6e

package/src/backup/backup-service.ts

@@ -0,0 +1,299 @@
+/**
+ * Backup / restore orchestration: thin, well-typed wrappers around
+ * `pg_dump` and `pg_restore`, plus binary detection, version pre-flight,
+ * and upload/prune against a storage backend.
+ *
+ * The pure argument- and policy-logic lives in `pg-tools.ts` and
+ * `retention.ts`; this module is the impure edge (spawns processes, talks
+ * to Postgres and storage) and is intentionally kept small.
+ */
+import fs from "fs";
+import os from "os";
+import path from "path";
+import { execa } from "execa";
+import type { StorageController } from "@rebasepro/server";
+import { resolveLocalBin } from "../cli-helpers";
+import {
+    BackupDestination,
+    buildBackupFilename,
+    buildPgDumpArgs,
+    buildPgRestoreArgs,
+    checkToolServerCompatibility,
+    joinStorageKey,
+    parsePgToolMajor,
+    parseBackupTimestamp,
+    serverVersionNumToMajor,
+    VersionCompatibility
+} from "./pg-tools";
+import { BackupObject, RetentionOptions, selectBackupsToPrune } from "./retention";
+
+export class BackupToolError extends Error {
+    constructor(message: string, readonly hint?: string) {
+        super(message);
+        this.name = "BackupToolError";
+    }
+}
+
+/** Locate `pg_dump` / `pg_restore`, honouring an env override. */
+export function resolvePgBinary(
+    tool: "pg_dump" | "pg_restore",
+    env: Record<string, string | undefined> = process.env
+): string | null {
+    const override = tool === "pg_dump" ? env.PG_DUMP_PATH : env.PG_RESTORE_PATH;
+    if (override && fs.existsSync(override)) return override;
+    return resolveLocalBin(tool);
+}
+
+/** Run `<bin> --version` and extract the major version. */
+export async function detectToolMajor(bin: string): Promise<number | null> {
+    try {
+        const { stdout } = await execa(bin, ["--version"]);
+        return parsePgToolMajor(stdout);
+    } catch {
+        return null;
+    }
+}
+
+/** Query the server for its major version via `server_version_num`. */
+export async function getServerVersionMajor(connectionString: string): Promise<number | null> {
+    const { Client } = await import("pg");
+    const client = new Client({ connectionString });
+    await client.connect();
+    try {
+        const res = await client.query("SHOW server_version_num");
+        const raw = res.rows?.[0]?.server_version_num;
+        return serverVersionNumToMajor(raw);
+    } finally {
+        await client.end();
+    }
+}
+
+export interface PreflightResult extends VersionCompatibility {
+    bin: string;
+    toolMajor: number | null;
+    serverMajor: number | null;
+}
+
+/**
+ * Verify the requested client tool exists and its major version is
+ * compatible with the live server. Throws {@link BackupToolError} — with a
+ * doctor-style hint — when the binary is missing.
+ */
+export async function preflight(
+    tool: "pg_dump" | "pg_restore",
+    connectionString: string,
+    env: Record<string, string | undefined> = process.env
+): Promise<PreflightResult> {
+    const bin = resolvePgBinary(tool, env);
+    if (!bin) {
+        throw new BackupToolError(
+            `Could not find the '${tool}' binary.`,
+            `Install the PostgreSQL client tools (e.g. 'brew install libpq' or 'apt-get install postgresql-client'), ` +
+            `or set ${tool === "pg_dump" ? "PG_DUMP_PATH" : "PG_RESTORE_PATH"} to its full path.`
+        );
+    }
+    const [toolMajor, serverMajor] = await Promise.all([
+        detectToolMajor(bin),
+        getServerVersionMajor(connectionString)
+    ]);
+    const compat = checkToolServerCompatibility(toolMajor, serverMajor);
+    return { bin, toolMajor, serverMajor, ...compat };
+}
+
+export interface BackupResult {
+    /** Absolute path of the produced dump file on local disk. */
+    localFile: string;
+    fileName: string;
+    sizeBytes: number;
+}
+
+/**
+ * Produce a custom-format dump on local disk. When `outDir` is omitted the
+ * file is written to the OS temp directory (used by the upload path, which
+ * cleans it up afterwards).
+ */
+export async function createDump(opts: {
+    connectionString: string;
+    dbName: string;
+    outDir?: string;
+    fileName?: string;
+    excludeSchemas?: string[];
+    noOwner?: boolean;
+    inheritStdio?: boolean;
+    env?: Record<string, string | undefined>;
+}): Promise<BackupResult> {
+    const env = opts.env ?? process.env;
+    const bin = resolvePgBinary("pg_dump", env);
+    if (!bin) {
+        throw new BackupToolError(
+            "Could not find the 'pg_dump' binary.",
+            "Install the PostgreSQL client tools or set PG_DUMP_PATH."
+        );
+    }
+
+    const fileName = opts.fileName ?? buildBackupFilename(opts.dbName);
+    const outDir = opts.outDir ?? fs.mkdtempSync(path.join(os.tmpdir(), "rebase-backup-"));
+    fs.mkdirSync(outDir, { recursive: true });
+    const localFile = path.join(outDir, fileName);
+
+    const args = buildPgDumpArgs({
+        connectionString: opts.connectionString,
+        outFile: localFile,
+        excludeSchemas: opts.excludeSchemas,
+        noOwner: opts.noOwner
+    });
+
+    await execa(bin, args, {
+        stdio: opts.inheritStdio ? "inherit" : "pipe",
+        env: { ...(env as Record<string, string>) }
+    });
+
+    const sizeBytes = fs.existsSync(localFile) ? fs.statSync(localFile).size : 0;
+    return { localFile, fileName, sizeBytes };
+}
+
+/**
+ * Restore a custom-format dump into the database named by
+ * `connectionString`. Destructive when `clean` is set (drops objects
+ * first). Never called automatically — the CLI gates it behind explicit
+ * confirmation.
+ */
+export async function restoreDump(opts: {
+    connectionString: string;
+    inputFile: string;
+    clean?: boolean;
+    noOwner?: boolean;
+    inheritStdio?: boolean;
+    env?: Record<string, string | undefined>;
+}): Promise<void> {
+    const env = opts.env ?? process.env;
+    const bin = resolvePgBinary("pg_restore", env);
+    if (!bin) {
+        throw new BackupToolError(
+            "Could not find the 'pg_restore' binary.",
+            "Install the PostgreSQL client tools or set PG_RESTORE_PATH."
+        );
+    }
+    const args = buildPgRestoreArgs({
+        connectionString: opts.connectionString,
+        inputFile: opts.inputFile,
+        clean: opts.clean,
+        noOwner: opts.noOwner
+    });
+    await execa(bin, args, {
+        stdio: opts.inheritStdio ? "inherit" : "pipe",
+        env: { ...(env as Record<string, string>) }
+    });
+}
+
+/**
+ * Create a database (if absent) by connecting to the maintenance
+ * `postgres` database. Used by `restore --create-db`.
+ */
+export async function ensureDatabaseExists(
+    adminConnectionString: string,
+    dbName: string
+): Promise<boolean> {
+    const { Client } = await import("pg");
+    const parsed = new URL(adminConnectionString);
+    parsed.pathname = "/postgres";
+    const client = new Client({ connectionString: parsed.toString() });
+    await client.connect();
+    try {
+        const existing = await client.query("SELECT 1 FROM pg_database WHERE datname = $1", [dbName]);
+        if (existing.rowCount && existing.rowCount > 0) {
+            return false;
+        }
+        // Identifier can't be parameterised; guard against injection.
+        if (!/^[a-zA-Z_][a-zA-Z0-9_]*$/.test(dbName)) {
+            throw new BackupToolError(`Invalid database name: "${dbName}"`);
+        }
+        await client.query(`CREATE DATABASE "${dbName}"`);
+        return true;
+    } finally {
+        await client.end();
+    }
+}
+
+/**
+ * Upload a local dump to object storage under the destination's prefix.
+ * Backups may contain secrets/PII, so the object is written with a
+ * private/octet-stream content type; bucket-level ACLs must stay private.
+ */
+export async function uploadBackup(
+    storage: StorageController,
+    localFile: string,
+    dest: Extract<BackupDestination, { kind: "s3" | "gcs" }>
+): Promise<{ key: string; storageUrl: string }> {
+    const fileName = path.basename(localFile);
+    const key = joinStorageKey(dest.prefix, fileName);
+    const buffer = fs.readFileSync(localFile);
+    const file = new File([buffer], fileName, { type: "application/octet-stream" });
+    const result = await storage.putObject({
+        file,
+        key,
+        bucket: dest.bucket,
+        metadata: { "rebase-backup": "1" }
+    });
+    return { key, storageUrl: result.storageUrl ?? `${dest.kind}://${dest.bucket}/${key}` };
+}
+
+/**
+ * List existing backups at a destination. For local destinations reads the
+ * directory; for object storage lists the prefix via the controller.
+ */
+export async function listBackups(
+    dest: BackupDestination,
+    storage?: StorageController
+): Promise<BackupObject[]> {
+    if (dest.kind === "local") {
+        if (!fs.existsSync(dest.path)) return [];
+        const stat = fs.statSync(dest.path);
+        const dir = stat.isDirectory() ? dest.path : path.dirname(dest.path);
+        return fs
+            .readdirSync(dir)
+            .filter((f) => f.endsWith(".dump"))
+            .map((f) => {
+                const full = path.join(dir, f);
+                const createdAt = parseBackupTimestamp(f) ?? fs.statSync(full).mtime;
+                return { key: full, createdAt };
+            })
+            .sort((a, b) => (b.createdAt?.getTime() ?? 0) - (a.createdAt?.getTime() ?? 0));
+    }
+
+    if (!storage) {
+        throw new BackupToolError(
+            `Listing ${dest.kind} backups requires a configured storage backend.`
+        );
+    }
+    const result = await storage.listObjects(dest.prefix ? `${dest.prefix}/` : "", {
+        bucket: dest.bucket,
+        maxResults: 1000
+    });
+    return result.items
+        .map((item) => item.fullPath)
+        .filter((key) => key.endsWith(".dump"))
+        .map((key) => ({ key, createdAt: parseBackupTimestamp(key) ?? undefined }))
+        .sort((a, b) => (b.createdAt?.getTime() ?? 0) - (a.createdAt?.getTime() ?? 0));
+}
+
+/**
+ * Apply a retention policy to a destination, deleting the backups selected
+ * by {@link selectBackupsToPrune}. Returns the keys that were removed.
+ */
+export async function pruneBackups(
+    dest: BackupDestination,
+    options: RetentionOptions,
+    storage?: StorageController
+): Promise<string[]> {
+    const backups = await listBackups(dest, storage);
+    const toDelete = selectBackupsToPrune(backups, options);
+    for (const key of toDelete) {
+        if (dest.kind === "local") {
+            if (fs.existsSync(key)) fs.unlinkSync(key);
+        } else if (storage) {
+            await storage.deleteObject(key, dest.bucket);
+        }
+    }
+    return toDelete;
+}

package/src/backup/index.ts

@@ -0,0 +1,12 @@
+/**
+ * Backup / restore for self-hosted rebase.
+ *
+ * - `pg-tools`      — pure argument/version/path helpers (unit-tested)
+ * - `retention`     — pure retention-pruning policy (unit-tested)
+ * - `backup-service`— pg_dump/pg_restore + storage orchestration
+ * - `backup-cron`   — scheduled backups for the server cron system
+ */
+export * from "./pg-tools";
+export * from "./retention";
+export * from "./backup-service";
+export * from "./backup-cron";

package/src/backup/pg-tools.ts

@@ -0,0 +1,231 @@
+/**
+ * Pure helpers for the backup/restore commands.
+ *
+ * Everything in this file is side-effect free so it can be unit-tested
+ * without a live Postgres server, matching the constraint that CI must
+ * not require a database.
+ */
+
+/**
+ * A parsed backup destination. `--out` (and the scheduled-backup config)
+ * accepts either a local filesystem path or an object-storage URL.
+ */
+export type BackupDestination =
+    | { kind: "local"; path: string }
+    | { kind: "s3"; bucket: string; prefix: string }
+    | { kind: "gcs"; bucket: string; prefix: string };
+
+/**
+ * Extract the database name from a Postgres connection string.
+ * Returns `null` when the URL has no database path (e.g. bare host).
+ */
+export function parseDbNameFromUrl(connectionString: string): string | null {
+    try {
+        const parsed = new URL(connectionString);
+        const name = parsed.pathname.replace(/^\//, "").trim();
+        return name.length > 0 ? name : null;
+    } catch {
+        // Fall back to a permissive regex for non-URL DSNs.
+        const match = connectionString.match(/\/([^/?]+)(\?|$)/);
+        return match && match[1] ? match[1] : null;
+    }
+}
+
+/**
+ * Swap the database name in a connection string, preserving credentials,
+ * host, port and query params. Used by `--target-db` / `--create-db` so a
+ * restore can target a fresh database instead of clobbering the live one.
+ */
+export function withDatabaseName(connectionString: string, dbName: string): string {
+    try {
+        const parsed = new URL(connectionString);
+        parsed.pathname = `/${dbName}`;
+        return parsed.toString();
+    } catch {
+        return connectionString.replace(/\/([^/?]+)(\?|$)/, `/${dbName}$2`);
+    }
+}
+
+/**
+ * Parse the major version out of `pg_dump --version` / `pg_restore --version`
+ * output, e.g. `"pg_dump (PostgreSQL) 16.2"` → `16`. Handles the pre-10
+ * `9.6.x` scheme (returns `9`) and beta strings like `"17beta1"`.
+ */
+export function parsePgToolMajor(versionOutput: string): number | null {
+    const match = versionOutput.match(/(\d+)(?:\.(\d+))?/);
+    if (!match) return null;
+    const major = Number(match[1]);
+    if (!Number.isFinite(major)) return null;
+    // Postgres 9.x used the first two numbers as the major (9.6, 9.4…).
+    if (major === 9 && match[2] !== undefined) {
+        return 9;
+    }
+    return major;
+}
+
+/**
+ * Convert `SELECT current_setting('server_version_num')` (e.g. `160002`)
+ * into a major version (`16`). Also accepts pre-10 encodings like `90603`
+ * → `9`.
+ */
+export function serverVersionNumToMajor(versionNum: number | string): number | null {
+    const num = typeof versionNum === "string" ? Number(versionNum) : versionNum;
+    if (!Number.isFinite(num) || num <= 0) return null;
+    if (num < 100000) {
+        // 9.x scheme: 90603 = 9.6.3
+        return Math.floor(num / 10000);
+    }
+    return Math.floor(num / 10000);
+}
+
+export interface VersionCompatibility {
+    compatible: boolean;
+    reason?: string;
+}
+
+/**
+ * pg_dump / pg_restore must be **at least** as new as the server they talk
+ * to. A newer client against an older server is supported; an older client
+ * against a newer server is not and produces corrupt or rejected output.
+ */
+export function checkToolServerCompatibility(
+    toolMajor: number | null,
+    serverMajor: number | null
+): VersionCompatibility {
+    if (toolMajor === null) {
+        return { compatible: false, reason: "Could not determine the client tool version." };
+    }
+    if (serverMajor === null) {
+        return { compatible: false, reason: "Could not determine the Postgres server version." };
+    }
+    if (toolMajor < serverMajor) {
+        return {
+            compatible: false,
+            reason:
+                `Client tool is Postgres ${toolMajor} but the server is Postgres ${serverMajor}. ` +
+                `pg_dump/pg_restore must be the same major version as the server or newer. ` +
+                `Install Postgres ${serverMajor} client tools.`
+        };
+    }
+    return { compatible: true };
+}
+
+/**
+ * Build a deterministic, sortable backup file name:
+ *   `rebase-<db>-<YYYYMMDD>T<HHMMSS>Z.dump`
+ *
+ * The UTC timestamp is embedded so retention pruning can recover the
+ * creation time from the object key alone, without extra metadata.
+ */
+export function buildBackupFilename(dbName: string, date: Date = new Date()): string {
+    const iso = date.toISOString(); // 2026-07-14T09:12:03.123Z
+    const stamp = iso.replace(/\.\d+Z$/, "Z").replace(/[-:]/g, "");
+    const safeDb = dbName.replace(/[^a-zA-Z0-9_-]/g, "_");
+    return `rebase-${safeDb}-${stamp}.dump`;
+}
+
+/**
+ * Recover the creation timestamp encoded in a backup file name by
+ * {@link buildBackupFilename}. Returns `null` for names that don't match,
+ * so foreign objects in a shared prefix are never pruned.
+ */
+export function parseBackupTimestamp(fileName: string): Date | null {
+    const base = fileName.split("/").pop() ?? fileName;
+    const match = base.match(/-(\d{8})T(\d{6})Z\.dump$/);
+    if (!match) return null;
+    const [, ymd, hms] = match;
+    const iso =
+        `${ymd.slice(0, 4)}-${ymd.slice(4, 6)}-${ymd.slice(6, 8)}` +
+        `T${hms.slice(0, 2)}:${hms.slice(2, 4)}:${hms.slice(4, 6)}Z`;
+    const date = new Date(iso);
+    return Number.isNaN(date.getTime()) ? null : date;
+}
+
+/**
+ * Parse a destination string into a structured {@link BackupDestination}.
+ * `s3://bucket/prefix` and `gs://bucket/prefix` map to object storage;
+ * anything else is treated as a local path.
+ */
+export function parseBackupDestination(out: string): BackupDestination {
+    const s3 = out.match(/^s3:\/\/([^/]+)\/?(.*)$/);
+    if (s3) {
+        return { kind: "s3", bucket: s3[1], prefix: stripTrailingSlash(s3[2]) };
+    }
+    const gcs = out.match(/^gs:\/\/([^/]+)\/?(.*)$/);
+    if (gcs) {
+        return { kind: "gcs", bucket: gcs[1], prefix: stripTrailingSlash(gcs[2]) };
+    }
+    return { kind: "local", path: out };
+}
+
+function stripTrailingSlash(s: string): string {
+    return s.replace(/\/+$/, "");
+}
+
+/**
+ * Join a storage prefix and a file name without producing a leading or
+ * doubled slash.
+ */
+export function joinStorageKey(prefix: string, fileName: string): string {
+    const clean = prefix.replace(/^\/+|\/+$/g, "");
+    return clean.length > 0 ? `${clean}/${fileName}` : fileName;
+}
+
+/**
+ * Assemble the `pg_dump` argument vector. Uses the custom format (`-Fc`),
+ * which is compressed and restorable selectively via `pg_restore`.
+ */
+export function buildPgDumpArgs(opts: {
+    connectionString: string;
+    outFile: string;
+    /** Extra schemas/tables to exclude, e.g. Atlas revision tables. */
+    excludeSchemas?: string[];
+    /** Number of parallel jobs (directory format only; ignored for -Fc). */
+    noOwner?: boolean;
+}): string[] {
+    const args = ["--format=custom", "--no-password", `--file=${opts.outFile}`];
+    if (opts.noOwner) {
+        args.push("--no-owner");
+    }
+    for (const schema of opts.excludeSchemas ?? []) {
+        args.push(`--exclude-schema=${schema}`);
+    }
+    args.push(opts.connectionString);
+    return args;
+}
+
+/**
+ * Assemble the `pg_restore` argument vector for a custom-format dump.
+ */
+export function buildPgRestoreArgs(opts: {
+    connectionString: string;
+    inputFile: string;
+    /** Drop objects before recreating them (destructive but idempotent). */
+    clean?: boolean;
+    /** Continue past individual errors instead of aborting. */
+    exitOnError?: boolean;
+    noOwner?: boolean;
+}): string[] {
+    const args = ["--format=custom", "--no-password", `--dbname=${opts.connectionString}`];
+    if (opts.clean) {
+        args.push("--clean", "--if-exists");
+    }
+    if (opts.noOwner) {
+        args.push("--no-owner");
+    }
+    if (opts.exitOnError) {
+        args.push("--exit-on-error");
+    }
+    args.push(opts.inputFile);
+    return args;
+}
+
+/**
+ * Resolve the Postgres connection string the backup commands should use,
+ * mirroring the precedence the branch command already relies on.
+ */
+export function resolveConnectionString(
+    env: Record<string, string | undefined>
+): string | null {
+    return env.DATABASE_URL || env.ADMIN_CONNECTION_STRING || null;
+}

package/src/backup/retention.ts

@@ -0,0 +1,75 @@
+/**
+ * Retention pruning for scheduled backups. Pure logic — no I/O — so it can
+ * be exhaustively unit-tested without touching a storage backend.
+ */
+import { parseBackupTimestamp } from "./pg-tools";
+
+export interface BackupObject {
+    /** Storage key / file name. */
+    key: string;
+    /**
+     * Creation time. Optional — when absent it is recovered from the
+     * timestamp encoded in the key by `buildBackupFilename`.
+     */
+    createdAt?: Date;
+}
+
+export interface RetentionOptions {
+    /** Delete backups older than this many days. `0`/negative disables age pruning. */
+    retentionDays?: number;
+    /**
+     * Always keep at least this many of the most recent backups, even if
+     * they are older than `retentionDays`. Guards against wiping every
+     * backup after a long outage. Default: 0 (no minimum).
+     */
+    keepMinimum?: number;
+    /** Reference "now" — injectable for deterministic tests. */
+    now?: Date;
+}
+
+/**
+ * Decide which backups to delete under the given retention policy.
+ *
+ * Rules, applied in order:
+ *  1. Objects whose key isn't a recognised rebase backup (no parseable
+ *     timestamp and no `createdAt`) are never selected — foreign files in a
+ *     shared bucket are left untouched.
+ *  2. The `keepMinimum` newest backups are always retained.
+ *  3. Of the remainder, any older than `retentionDays` are selected for
+ *     deletion.
+ *
+ * Returns the keys to delete, oldest first.
+ */
+export function selectBackupsToPrune(
+    backups: BackupObject[],
+    options: RetentionOptions
+): string[] {
+    const retentionDays = options.retentionDays ?? 0;
+    const keepMinimum = Math.max(0, options.keepMinimum ?? 0);
+    const now = options.now ?? new Date();
+
+    if (retentionDays <= 0) {
+        return [];
+    }
+
+    // Resolve a timestamp for every object; drop the ones we can't date.
+    const dated = backups
+        .map((b) => {
+            const createdAt = b.createdAt ?? parseBackupTimestamp(b.key);
+            return createdAt ? { key: b.key, createdAt } : null;
+        })
+        .filter((b): b is { key: string; createdAt: Date } => b !== null);
+
+    // Newest first so the first `keepMinimum` are the protected ones.
+    dated.sort((a, b) => b.createdAt.getTime() - a.createdAt.getTime());
+
+    const cutoff = now.getTime() - retentionDays * 24 * 60 * 60 * 1000;
+
+    const toDelete = dated
+        .slice(keepMinimum)
+        .filter((b) => b.createdAt.getTime() < cutoff);
+
+    // Return oldest first for predictable, readable logs.
+    toDelete.sort((a, b) => a.createdAt.getTime() - b.createdAt.getTime());
+    return toDelete.map((b) => b.key);
+}