@useatlas/create 0.0.6 → 0.0.7

package/templates/docker/ee/src/backups/engine.ts

@@ -0,0 +1,355 @@
+/**
+ * Backup engine — pg_dump-based backup with gzip compression.
+ *
+ * Creates compressed backups of the internal PostgreSQL database
+ * and stores them to a configurable local directory.
+ *
+ * Enterprise-gated via requireEnterprise("backups").
+ */
+
+import { spawn } from "child_process";
+import { createWriteStream } from "fs";
+import { mkdir, stat, unlink, readdir } from "fs/promises";
+import { join, dirname } from "path";
+import { createGzip } from "zlib";
+import { pipeline } from "stream/promises";
+import { requireEnterprise } from "../index";
+import { hasInternalDB, internalQuery } from "@atlas/api/lib/db/internal";
+import { createLogger } from "@atlas/api/lib/logger";
+import type { BackupStatus } from "@useatlas/types";
+
+const log = createLogger("ee:backups-engine");
+
+// ---------------------------------------------------------------------------
+// Table bootstrap — idempotent, runs on first use
+// ---------------------------------------------------------------------------
+
+let _tableReady = false;
+
+export async function ensureTable(): Promise<void> {
+  if (_tableReady) return;
+  if (!hasInternalDB()) {
+    throw new Error("Internal database not configured — backups require DATABASE_URL");
+  }
+
+  await internalQuery(
+    `CREATE TABLE IF NOT EXISTS backups (
+       id                    TEXT PRIMARY KEY DEFAULT gen_random_uuid()::text,
+       created_at            TIMESTAMPTZ NOT NULL DEFAULT now(),
+       size_bytes            BIGINT,
+       status                TEXT NOT NULL DEFAULT 'in_progress',
+       storage_path          TEXT NOT NULL,
+       retention_expires_at  TIMESTAMPTZ NOT NULL,
+       error_message         TEXT
+     )`,
+  );
+  await internalQuery(
+    `CREATE INDEX IF NOT EXISTS idx_backups_status ON backups (status, created_at DESC)`,
+  );
+
+  // Backup config table — single row keyed by '_default'
+  await internalQuery(
+    `CREATE TABLE IF NOT EXISTS backup_config (
+       id              TEXT PRIMARY KEY DEFAULT '_default',
+       schedule        TEXT NOT NULL DEFAULT '0 3 * * *',
+       retention_days  INT NOT NULL DEFAULT 30,
+       storage_path    TEXT NOT NULL DEFAULT './backups',
+       updated_at      TIMESTAMPTZ NOT NULL DEFAULT now()
+     )`,
+  );
+
+  // Seed default config
+  const envSchedule = process.env.ATLAS_BACKUP_SCHEDULE ?? "0 3 * * *";
+  const envRetention = parseInt(process.env.ATLAS_BACKUP_RETENTION_DAYS ?? "30", 10) || 30;
+  const envStorage = process.env.ATLAS_BACKUP_STORAGE_PATH ?? "./backups";
+  await internalQuery(
+    `INSERT INTO backup_config (id, schedule, retention_days, storage_path)
+     VALUES ('_default', $1, $2, $3)
+     ON CONFLICT (id) DO NOTHING`,
+    [envSchedule, envRetention, envStorage],
+  );
+
+  _tableReady = true;
+}
+
+/** @internal Reset table-ready flag — for testing only. */
+export function _resetTableReady(): void {
+  _tableReady = false;
+}
+
+// ---------------------------------------------------------------------------
+// Configuration
+// ---------------------------------------------------------------------------
+
+export type BackupConfigRow = {
+  schedule: string;
+  retention_days: number;
+  storage_path: string;
+};
+
+export async function getBackupConfig(): Promise<BackupConfigRow> {
+  await ensureTable();
+  const rows = await internalQuery<{
+    schedule: string;
+    retention_days: number;
+    storage_path: string;
+  }>(
+    `SELECT schedule, retention_days, storage_path FROM backup_config WHERE id = '_default'`,
+  );
+  if (!rows[0]) {
+    log.warn("Backup config row missing from database — using defaults");
+    return { schedule: "0 3 * * *", retention_days: 30, storage_path: "./backups" };
+  }
+  return rows[0];
+}
+
+export async function updateBackupConfig(
+  config: { schedule?: string; retentionDays?: number; storagePath?: string },
+): Promise<void> {
+  await ensureTable();
+  const current = await getBackupConfig();
+  const schedule = config.schedule ?? current.schedule;
+  const retentionDays = config.retentionDays ?? current.retention_days;
+  const storagePath = config.storagePath ?? current.storage_path;
+
+  await internalQuery(
+    `INSERT INTO backup_config (id, schedule, retention_days, storage_path, updated_at)
+     VALUES ('_default', $1, $2, $3, now())
+     ON CONFLICT (id) DO UPDATE SET
+       schedule = $1,
+       retention_days = $2,
+       storage_path = $3,
+       updated_at = now()`,
+    [schedule, retentionDays, storagePath],
+  );
+}
+
+// ---------------------------------------------------------------------------
+// Backup creation
+// ---------------------------------------------------------------------------
+
+/**
+ * Parse DATABASE_URL into pg_dump connection arguments.
+ * Extracts host, port, user, dbname from the URL. Password is passed
+ * via PGPASSWORD env var to avoid shell exposure.
+ */
+function parseDatabaseUrl(url: string): { args: string[]; password: string | undefined } {
+  const parsed = new URL(url);
+  const args: string[] = [];
+
+  if (parsed.hostname) args.push("-h", parsed.hostname);
+  if (parsed.port) args.push("-p", parsed.port);
+  if (parsed.username) args.push("-U", parsed.username);
+
+  // Database name is the path without leading slash
+  const dbName = parsed.pathname.replace(/^\//, "");
+  if (dbName) args.push("-d", dbName);
+
+  return { args, password: parsed.password ? decodeURIComponent(parsed.password) : undefined };
+}
+
+/**
+ * Create a backup of the internal database.
+ *
+ * Uses pg_dump with plain-text format, then gzip-compresses the output.
+ * Records the backup in the internal backups table.
+ */
+export async function createBackup(): Promise<{
+  id: string;
+  storagePath: string;
+  sizeBytes: number;
+  status: BackupStatus;
+}> {
+  requireEnterprise("backups");
+  await ensureTable();
+
+  const databaseUrl = process.env.DATABASE_URL;
+  if (!databaseUrl) {
+    throw new Error("DATABASE_URL is not set — cannot create backup");
+  }
+
+  const config = await getBackupConfig();
+  const timestamp = new Date().toISOString().replace(/[:.]/g, "-");
+  const filename = `atlas-backup-${timestamp}.sql.gz`;
+  const storagePath = join(config.storage_path, filename);
+  const retentionExpiresAt = new Date(Date.now() + config.retention_days * 24 * 60 * 60 * 1000).toISOString();
+
+  // Insert in-progress record
+  const rows = await internalQuery<{ id: string }>(
+    `INSERT INTO backups (status, storage_path, retention_expires_at)
+     VALUES ('in_progress', $1, $2)
+     RETURNING id`,
+    [storagePath, retentionExpiresAt],
+  );
+  const backupId = rows[0].id;
+
+  try {
+    // Ensure storage directory exists
+    await mkdir(dirname(storagePath), { recursive: true });
+
+    const { args, password } = parseDatabaseUrl(databaseUrl);
+
+    // Run pg_dump → gzip → file
+    const pgDump = spawn("pg_dump", [...args, "--format=plain"], {
+      env: { ...process.env, PGPASSWORD: password ?? "" },
+      stdio: ["ignore", "pipe", "pipe"],
+    });
+
+    const outStream = createWriteStream(storagePath);
+    const gzip = createGzip();
+
+    // Collect stderr for error reporting
+    let stderr = "";
+    pgDump.stderr.on("data", (chunk: Buffer) => {
+      stderr += chunk.toString();
+    });
+
+    await pipeline(pgDump.stdout, gzip, outStream);
+
+    const exitCode = await new Promise<number>((resolve) => {
+      pgDump.on("close", resolve);
+    });
+
+    if (exitCode !== 0) {
+      throw new Error(`pg_dump exited with code ${exitCode}: ${stderr.slice(0, 500)}`);
+    }
+
+    // Get file size
+    const fileStat = await stat(storagePath);
+
+    // Mark as completed
+    await internalQuery(
+      `UPDATE backups SET status = 'completed', size_bytes = $1 WHERE id = $2`,
+      [fileStat.size, backupId],
+    );
+
+    log.info({ backupId, storagePath, sizeBytes: fileStat.size }, "Backup completed successfully");
+    return { id: backupId, storagePath, sizeBytes: fileStat.size, status: "completed" };
+  } catch (err) {
+    const errorMessage = err instanceof Error ? err.message : String(err);
+    await internalQuery(
+      `UPDATE backups SET status = 'failed', error_message = $1 WHERE id = $2`,
+      [errorMessage.slice(0, 2000), backupId],
+    ).catch((updateErr) => {
+      log.warn(
+        { err: updateErr instanceof Error ? updateErr.message : String(updateErr), backupId },
+        "Failed to update backup status to failed",
+      );
+    });
+
+    log.error({ err: err instanceof Error ? err : new Error(String(err)), backupId }, "Backup failed");
+    throw err;
+  }
+}
+
+// ---------------------------------------------------------------------------
+// List / purge
+// ---------------------------------------------------------------------------
+
+export type BackupRow = {
+  id: string;
+  created_at: string;
+  size_bytes: string | null;
+  status: BackupStatus;
+  storage_path: string;
+  retention_expires_at: string;
+  error_message: string | null;
+};
+
+type BackupRowQuery = {
+  id: string;
+  created_at: string;
+  size_bytes: string | null;
+  status: string;
+  storage_path: string;
+  retention_expires_at: string;
+  error_message: string | null;
+};
+
+export async function listBackups(limit = 50): Promise<BackupRow[]> {
+  await ensureTable();
+  const rows = await internalQuery<BackupRowQuery>(
+    `SELECT id, created_at::text, size_bytes::text, status, storage_path, retention_expires_at::text, error_message
+     FROM backups
+     ORDER BY created_at DESC
+     LIMIT $1`,
+    [limit],
+  );
+  return rows as BackupRow[];
+}
+
+export async function getBackupById(id: string): Promise<BackupRow | null> {
+  await ensureTable();
+  const rows = await internalQuery<BackupRowQuery>(
+    `SELECT id, created_at::text, size_bytes::text, status, storage_path, retention_expires_at::text, error_message
+     FROM backups WHERE id = $1`,
+    [id],
+  );
+  return (rows[0] as BackupRow | undefined) ?? null;
+}
+
+/**
+ * Delete expired backups — removes both the DB record and the file on disk.
+ */
+export async function purgeExpiredBackups(): Promise<number> {
+  await ensureTable();
+
+  const expired = await internalQuery<{ id: string; storage_path: string }>(
+    `SELECT id, storage_path FROM backups WHERE retention_expires_at < now()`,
+  );
+
+  let purged = 0;
+  for (const row of expired) {
+    try {
+      await unlink(row.storage_path);
+    } catch (err) {
+      const code = err instanceof Error && "code" in err
+        ? (err as NodeJS.ErrnoException).code
+        : undefined;
+      if (code !== "ENOENT") {
+        log.warn(
+          { err: err instanceof Error ? err.message : String(err), backupId: row.id, path: row.storage_path },
+          "Could not delete backup file — skipping DB record deletion to avoid orphan",
+        );
+        continue;
+      }
+      // ENOENT is fine — file already removed
+    }
+
+    try {
+      await internalQuery(`DELETE FROM backups WHERE id = $1`, [row.id]);
+      purged++;
+    } catch (err) {
+      log.warn(
+        { err: err instanceof Error ? err.message : String(err), backupId: row.id },
+        "Failed to delete backup DB record — will retry on next purge cycle",
+      );
+    }
+  }
+
+  if (purged > 0) {
+    log.info({ count: purged }, "Purged expired backups");
+  }
+
+  return purged;
+}
+
+/**
+ * List files in the backup storage directory for verification purposes.
+ */
+export async function listStorageFiles(): Promise<string[]> {
+  const config = await getBackupConfig();
+  try {
+    const files = await readdir(config.storage_path);
+    return files.filter((f) => f.endsWith(".sql.gz"));
+  } catch (err) {
+    if (err instanceof Error && "code" in err && (err as NodeJS.ErrnoException).code === "ENOENT") {
+      return [];
+    }
+    log.warn(
+      { err: err instanceof Error ? err.message : String(err), path: config.storage_path },
+      "Failed to list backup storage directory",
+    );
+    throw err;
+  }
+}

package/templates/docker/ee/src/backups/index.ts

@@ -0,0 +1,26 @@
+/**
+ * Backup and disaster recovery — enterprise feature.
+ *
+ * Exports:
+ * - createBackup() / listBackups() / getBackupById() — backup operations
+ * - verifyBackup() — integrity verification
+ * - requestRestore() / executeRestore() — restore with confirmation
+ * - startScheduler() / stopScheduler() — automated cron backups
+ * - getBackupConfig() / updateBackupConfig() — configuration
+ * - purgeExpiredBackups() — retention enforcement
+ */
+
+export {
+  createBackup,
+  listBackups,
+  getBackupById,
+  getBackupConfig,
+  updateBackupConfig,
+  purgeExpiredBackups,
+} from "./engine";
+
+export { verifyBackup } from "./verify";
+
+export { requestRestore, executeRestore } from "./restore";
+
+export { startScheduler, stopScheduler } from "./scheduler";

package/templates/docker/ee/src/backups/restore.ts

@@ -0,0 +1,169 @@
+/**
+ * Backup restore — psql-based restore with safety checks.
+ *
+ * Before restoring, creates a pre-restore backup so the operation
+ * is reversible. Requires a confirmation token to proceed.
+ *
+ * Enterprise-gated via requireEnterprise("backups").
+ */
+
+import { spawn } from "child_process";
+import { createReadStream } from "fs";
+import { createGunzip } from "zlib";
+import { pipeline } from "stream/promises";
+import { requireEnterprise } from "../index";
+import { hasInternalDB } from "@atlas/api/lib/db/internal";
+import { createLogger } from "@atlas/api/lib/logger";
+import { createBackup, getBackupById, ensureTable } from "./engine";
+
+const log = createLogger("ee:backups-restore");
+
+// In-memory token store — short-lived confirmation tokens.
+// Token is generated when a restore is requested, and must be
+// echoed back to confirm. This prevents accidental restores.
+const pendingRestores = new Map<string, { backupId: string; expiresAt: number }>();
+
+/**
+ * Generate a confirmation token for a restore operation.
+ * The token must be passed back to `executeRestore()` within 5 minutes.
+ */
+export async function requestRestore(backupId: string): Promise<{ confirmationToken: string; message: string }> {
+  requireEnterprise("backups");
+  await ensureTable();
+
+  if (!hasInternalDB()) {
+    throw new Error("Internal database not configured — cannot restore backup");
+  }
+
+  const backup = await getBackupById(backupId);
+  if (!backup) {
+    throw new Error("Backup not found");
+  }
+
+  if (backup.status !== "completed" && backup.status !== "verified") {
+    throw new Error(`Cannot restore backup with status "${backup.status}" — only completed or verified backups can be restored`);
+  }
+
+  const token = crypto.randomUUID();
+  pendingRestores.set(token, { backupId, expiresAt: Date.now() + 5 * 60 * 1000 });
+
+  // Cleanup expired tokens
+  for (const [key, value] of pendingRestores) {
+    if (value.expiresAt < Date.now()) pendingRestores.delete(key);
+  }
+
+  return {
+    confirmationToken: token,
+    message: `Restore requested for backup ${backupId}. Pass the confirmation token to confirm. Token expires in 5 minutes. A pre-restore backup will be created automatically.`,
+  };
+}
+
+/**
+ * Execute the restore operation after confirmation.
+ *
+ * Steps:
+ * 1. Validate the confirmation token
+ * 2. Create a pre-restore backup (safety net)
+ * 3. Decompress and pipe into psql
+ */
+export async function executeRestore(
+  confirmationToken: string,
+): Promise<{ restored: boolean; preRestoreBackupId: string; message: string }> {
+  requireEnterprise("backups");
+
+  const pending = pendingRestores.get(confirmationToken);
+  if (!pending) {
+    throw new Error("Invalid or expired confirmation token");
+  }
+
+  if (pending.expiresAt < Date.now()) {
+    pendingRestores.delete(confirmationToken);
+    throw new Error("Confirmation token has expired — request a new one");
+  }
+
+  pendingRestores.delete(confirmationToken);
+  const { backupId } = pending;
+
+  const databaseUrl = process.env.DATABASE_URL;
+  if (!databaseUrl) {
+    throw new Error("DATABASE_URL is not set — cannot restore backup");
+  }
+
+  const backup = await getBackupById(backupId);
+  if (!backup) {
+    throw new Error("Backup not found — it may have been purged");
+  }
+
+  log.warn({ backupId }, "Starting database restore — creating pre-restore backup first");
+
+  // Step 1: Create pre-restore safety backup
+  let preRestoreBackupId: string;
+  try {
+    const preRestore = await createBackup();
+    preRestoreBackupId = preRestore.id;
+    log.info({ preRestoreBackupId }, "Pre-restore backup created");
+  } catch (err) {
+    log.error(
+      { err: err instanceof Error ? err : new Error(String(err)), backupId },
+      "Failed to create pre-restore backup — aborting restore",
+    );
+    throw new Error(`Pre-restore backup failed: ${err instanceof Error ? err.message : String(err)}`, { cause: err });
+  }
+
+  // Step 2: Restore from backup
+  try {
+    const parsed = new URL(databaseUrl);
+    const psqlArgs: string[] = [];
+
+    if (parsed.hostname) psqlArgs.push("-h", parsed.hostname);
+    if (parsed.port) psqlArgs.push("-p", parsed.port);
+    if (parsed.username) psqlArgs.push("-U", parsed.username);
+    const dbName = parsed.pathname.replace(/^\//, "");
+    if (dbName) psqlArgs.push("-d", dbName);
+
+    // Add flags: single transaction and stop on error
+    psqlArgs.push("--single-transaction", "--set", "ON_ERROR_STOP=on");
+
+    const password = parsed.password ? decodeURIComponent(parsed.password) : "";
+
+    const psql = spawn("psql", psqlArgs, {
+      env: { ...process.env, PGPASSWORD: password },
+      stdio: ["pipe", "pipe", "pipe"],
+    });
+
+    let stderr = "";
+    psql.stderr.on("data", (chunk: Buffer) => {
+      stderr += chunk.toString();
+    });
+
+    const input = createReadStream(backup.storage_path);
+    const gunzip = createGunzip();
+
+    await pipeline(input, gunzip, psql.stdin);
+
+    const exitCode = await new Promise<number>((resolve) => {
+      psql.on("close", resolve);
+    });
+
+    if (exitCode !== 0) {
+      throw new Error(`psql exited with code ${exitCode}: ${stderr.slice(0, 500)}`);
+    }
+
+    log.info({ backupId, preRestoreBackupId }, "Database restore completed successfully");
+    return {
+      restored: true,
+      preRestoreBackupId,
+      message: `Database restored from backup ${backupId}. Pre-restore backup saved as ${preRestoreBackupId}.`,
+    };
+  } catch (err) {
+    log.error(
+      { err: err instanceof Error ? err : new Error(String(err)), backupId, preRestoreBackupId },
+      "Database restore failed — pre-restore backup is available for recovery",
+    );
+    throw new Error(
+      `Restore failed: ${err instanceof Error ? err.message : String(err)}. ` +
+      `Pre-restore backup ${preRestoreBackupId} is available for recovery.`,
+      { cause: err },
+    );
+  }
+}

package/templates/docker/ee/src/backups/scheduler.ts

@@ -0,0 +1,153 @@
+/**
+ * Backup scheduler — cron-based automated backups.
+ *
+ * Evaluates a cron expression to determine when the next backup
+ * should run. The scheduler is designed to be called periodically
+ * (e.g., every minute via setInterval) and will trigger a backup
+ * when the cron expression matches the current time.
+ *
+ * Enterprise-gated via requireEnterprise("backups").
+ */
+
+import { requireEnterprise } from "../index";
+import { createLogger } from "@atlas/api/lib/logger";
+import { createBackup, getBackupConfig, purgeExpiredBackups, ensureTable } from "./engine";
+
+const log = createLogger("ee:backups-scheduler");
+
+let _schedulerInterval: ReturnType<typeof setInterval> | null = null;
+let _lastRunMinute = -1;
+
+/**
+ * Parse a cron expression and check if it matches the current time.
+ * Supports standard 5-field cron: minute hour day-of-month month day-of-week.
+ */
+function cronMatchesNow(expression: string): boolean {
+  const parts = expression.trim().split(/\s+/);
+  if (parts.length !== 5) {
+    log.warn({ expression }, "Invalid cron expression — expected 5 fields. Scheduled backups will not run.");
+    return false;
+  }
+
+  const now = new Date();
+  const fields = [
+    now.getUTCMinutes(),  // 0-59
+    now.getUTCHours(),    // 0-23
+    now.getUTCDate(),     // 1-31
+    now.getUTCMonth() + 1, // 1-12
+    now.getUTCDay(),      // 0-6 (Sun=0)
+  ];
+
+  return parts.every((part, i) => fieldMatches(part, fields[i]));
+}
+
+function fieldMatches(pattern: string, value: number): boolean {
+  if (pattern === "*") return true;
+
+  // Handle step values: */N or N-M/S
+  if (pattern.includes("/")) {
+    const [range, stepStr] = pattern.split("/");
+    const step = parseInt(stepStr, 10);
+    if (isNaN(step) || step <= 0) return false;
+    if (range === "*") return value % step === 0;
+    // Range with step: e.g. 1-30/2
+    const [startStr, endStr] = range.split("-");
+    const start = parseInt(startStr, 10);
+    const end = endStr ? parseInt(endStr, 10) : start;
+    if (isNaN(start) || isNaN(end)) return false;
+    return value >= start && value <= end && (value - start) % step === 0;
+  }
+
+  // Handle comma-separated values: 1,5,10
+  if (pattern.includes(",")) {
+    return pattern.split(",").some((p) => fieldMatches(p.trim(), value));
+  }
+
+  // Handle ranges: N-M
+  if (pattern.includes("-")) {
+    const [startStr, endStr] = pattern.split("-");
+    const start = parseInt(startStr, 10);
+    const end = parseInt(endStr, 10);
+    if (isNaN(start) || isNaN(end)) return false;
+    return value >= start && value <= end;
+  }
+
+  // Single value
+  return parseInt(pattern, 10) === value;
+}
+
+/**
+ * Check if a backup should run now and execute it.
+ * Called once per minute by the scheduler interval.
+ */
+async function tick(): Promise<void> {
+  const now = new Date();
+  const currentMinute = now.getUTCHours() * 60 + now.getUTCMinutes();
+
+  // Prevent double-execution in the same minute
+  if (currentMinute === _lastRunMinute) return;
+
+  try {
+    const config = await getBackupConfig();
+
+    if (!cronMatchesNow(config.schedule)) return;
+
+    _lastRunMinute = currentMinute;
+    log.info({ schedule: config.schedule }, "Scheduled backup triggered");
+
+    await createBackup();
+
+    // Purge expired backups after successful backup
+    await purgeExpiredBackups();
+  } catch (err) {
+    log.error(
+      { err: err instanceof Error ? err : new Error(String(err)) },
+      "Scheduled backup failed",
+    );
+  }
+}
+
+/**
+ * Start the backup scheduler. Runs a check every 60 seconds.
+ * Idempotent — calling multiple times is safe.
+ */
+export async function startScheduler(): Promise<void> {
+  requireEnterprise("backups");
+  await ensureTable();
+
+  if (_schedulerInterval) return;
+
+  const config = await getBackupConfig();
+  log.info({ schedule: config.schedule, retentionDays: config.retention_days }, "Backup scheduler started");
+
+  // Check immediately, then every 60 seconds
+  tick().catch((err) => {
+    log.error(
+      { err: err instanceof Error ? err : new Error(String(err)) },
+      "Initial scheduler tick failed",
+    );
+  });
+
+  _schedulerInterval = setInterval(() => {
+    tick().catch((err) => {
+      log.error(
+        { err: err instanceof Error ? err : new Error(String(err)) },
+        "Scheduler tick failed",
+      );
+    });
+  }, 60_000);
+}
+
+/**
+ * Stop the backup scheduler.
+ */
+export function stopScheduler(): void {
+  if (_schedulerInterval) {
+    clearInterval(_schedulerInterval);
+    _schedulerInterval = null;
+    log.info("Backup scheduler stopped");
+  }
+}
+
+/** @internal — for testing */
+export { cronMatchesNow as _cronMatchesNow };