@joystick.js/db-canary 0.0.0-canary.2209

package/src/server/lib/backup_manager.js

@@ -0,0 +1,650 @@
+/**
+ * @fileoverview Backup and restore management system for JoystickDB with S3 integration.
+ * Provides comprehensive backup creation, restoration, scheduling, and cleanup functionality
+ * with support for AWS S3 storage, checksum verification, and automated retention policies.
+ * Includes sparse backup compression, integrity verification, and graceful error handling.
+ */
+
+import { S3Client, PutObjectCommand, GetObjectCommand, ListObjectsV2Command, DeleteObjectCommand } from '@aws-sdk/client-s3';
+import { createReadStream, createWriteStream, existsSync, unlinkSync, statSync } from 'fs';
+import { mkdir, rm } from 'fs/promises';
+import { resolve, join } from 'path';
+import { spawn } from 'child_process';
+import { createHash } from 'crypto';
+import { get_settings } from './load_settings.js';
+import { get_database } from './query_engine.js';
+import create_logger from './logger.js';
+
+const { create_context_logger } = create_logger('backup_manager');
+
+/** @type {S3Client|null} AWS S3 client instance for backup operations */
+let s3_client = null;
+
+/** @type {NodeJS.Timeout|null} Timer for scheduled backup operations */
+let backup_schedule_timer = null;
+
+/**
+ * Initializes and returns the S3 client with bucket configuration.
+ * Creates a new S3 client instance if one doesn't exist, using settings from configuration.
+ * Supports custom endpoints for S3-compatible services.
+ * @returns {Object} Object containing S3 client and bucket name
+ * @returns {S3Client} returns.client - Configured S3 client instance
+ * @returns {string} returns.bucket - S3 bucket name for backups
+ * @throws {Error} When S3 configuration is missing or invalid
+ */
+const get_s3_client = () => {
+  const log = create_context_logger();
+  
+  try {
+    const settings = get_settings();
+    
+    if (!settings.s3) {
+      throw new Error('S3 configuration not found in settings');
+    }
+    
+    const { bucket, region, access_key, secret_key, endpoint } = settings.s3;
+    
+    if (!bucket || !region || !access_key || !secret_key) {
+      throw new Error('Missing required S3 configuration: bucket, region, access_key, secret_key');
+    }
+    
+    if (!s3_client) {
+      const client_config = {
+        region,
+        credentials: {
+          accessKeyId: access_key,
+          secretAccessKey: secret_key
+        }
+      };
+      
+      if (endpoint && endpoint !== `https://s3.${region}.amazonaws.com`) {
+        client_config.endpoint = endpoint;
+        client_config.forcePathStyle = true;
+      }
+      
+      s3_client = new S3Client(client_config);
+      log.info('S3 client initialized', { bucket, region, endpoint: endpoint || 'default' });
+    }
+    
+    return { client: s3_client, bucket };
+  } catch (error) {
+    log.error('Failed to initialize S3 client', { error: error.message });
+    throw error;
+  }
+};
+
+/**
+ * Tests the S3 connection by attempting to list objects in the configured bucket.
+ * Validates that the S3 client can successfully communicate with the storage service.
+ * @returns {Promise<boolean>} True if connection test succeeds
+ * @throws {Error} When S3 connection fails or configuration is invalid
+ */
+const test_s3_connection = async () => {
+  const log = create_context_logger();
+  
+  try {
+    const { client, bucket } = get_s3_client();
+    
+    const command = new ListObjectsV2Command({
+      Bucket: bucket,
+      MaxKeys: 1
+    });
+    
+    await client.send(command);
+    log.info('S3 connection test successful', { bucket });
+    return true;
+  } catch (error) {
+    log.error('S3 connection test failed', { error: error.message });
+    throw new Error(`S3 connection failed: ${error.message}`);
+  }
+};
+
+/**
+ * Calculates SHA256 checksum for a file to verify integrity.
+ * Streams the file content to avoid loading large files into memory.
+ * @param {string} file_path - Path to the file to calculate checksum for
+ * @returns {Promise<string>} SHA256 checksum as hexadecimal string
+ * @throws {Error} When file cannot be read or checksum calculation fails
+ */
+const calculate_sha256 = (file_path) => {
+  return new Promise((resolve, reject) => {
+    const hash = createHash('sha256');
+    const stream = createReadStream(file_path);
+    
+    stream.on('data', (data) => hash.update(data));
+    stream.on('end', () => resolve(hash.digest('hex')));
+    stream.on('error', reject);
+  });
+};
+
+/**
+ * Creates a compressed backup of the database and uploads it to S3.
+ * Generates a timestamped tar.gz archive with sparse file support, calculates checksums,
+ * uploads to S3 with metadata, and cleans up temporary files.
+ * @returns {Promise<Object>} Backup creation result
+ * @returns {string} returns.filename - Generated backup filename
+ * @returns {number} returns.size_bytes - Backup file size in bytes
+ * @returns {string} returns.checksum - SHA256 checksum of backup file
+ * @returns {string} returns.created_at - ISO timestamp of backup creation
+ * @returns {number} returns.duration_ms - Time taken to create backup in milliseconds
+ * @returns {string} returns.s3_location - S3 URI of uploaded backup
+ * @throws {Error} When backup creation, compression, or upload fails
+ */
+const create_backup = async () => {
+  const log = create_context_logger();
+  const backup_start = Date.now();
+  
+  try {
+    // NOTE: Ensure S3 is configured.
+    const { client, bucket } = get_s3_client();
+    
+    // NOTE: Create temporary directory.
+    const temp_dir = resolve('./temp_backup');
+    await mkdir(temp_dir, { recursive: true });
+    
+    // NOTE: Generate backup filename.
+    const timestamp = new Date().toISOString().replace(/[:.]/g, '-');
+    const backup_filename = `joystickdb-backup-${timestamp}.tar.gz`;
+    const temp_backup_path = join(temp_dir, backup_filename);
+    
+    log.info('Starting backup creation', { backup_filename, temp_backup_path });
+    
+    // NOTE: Create compressed sparse backup.
+    const tar_process = spawn('tar', [
+      '--sparse',
+      '-czf',
+      temp_backup_path,
+      '-C',
+      './data',
+      '.'
+    ], {
+      stdio: ['pipe', 'pipe', 'pipe']
+    });
+    
+    let tar_output = '';
+    let tar_error = '';
+    
+    tar_process.stdout.on('data', (data) => {
+      tar_output += data.toString();
+    });
+    
+    tar_process.stderr.on('data', (data) => {
+      tar_error += data.toString();
+    });
+    
+    const tar_exit_code = await new Promise((resolve) => {
+      tar_process.on('close', resolve);
+    });
+    
+    if (tar_exit_code !== 0) {
+      throw new Error(`Tar process failed with exit code ${tar_exit_code}: ${tar_error}`);
+    }
+    
+    // NOTE: Calculate checksum.
+    const checksum = await calculate_sha256(temp_backup_path);
+    const backup_stats = statSync(temp_backup_path);
+    
+    log.info('Backup file created', {
+      backup_filename,
+      size_bytes: backup_stats.size,
+      size_mb: Math.round(backup_stats.size / 1024 / 1024),
+      checksum
+    });
+    
+    // NOTE: Upload to S3.
+    const upload_stream = createReadStream(temp_backup_path);
+    
+    const upload_command = new PutObjectCommand({
+      Bucket: bucket,
+      Key: backup_filename,
+      Body: upload_stream,
+      Metadata: {
+        checksum,
+        created_at: new Date().toISOString(),
+        size_bytes: backup_stats.size.toString()
+      }
+    });
+    
+    await client.send(upload_command);
+    
+    // NOTE: Clean up temporary file.
+    unlinkSync(temp_backup_path);
+    await rm(temp_dir, { recursive: true, force: true });
+    
+    const backup_duration = Date.now() - backup_start;
+    
+    log.info('Backup completed successfully', {
+      backup_filename,
+      duration_ms: backup_duration,
+      size_mb: Math.round(backup_stats.size / 1024 / 1024),
+      checksum,
+      s3_bucket: bucket
+    });
+    
+    return {
+      filename: backup_filename,
+      size_bytes: backup_stats.size,
+      checksum,
+      created_at: new Date().toISOString(),
+      duration_ms: backup_duration,
+      s3_location: `s3://${bucket}/${backup_filename}`
+    };
+  } catch (error) {
+    log.error('Backup creation failed', { error: error.message });
+    
+    // NOTE: Clean up on failure.
+    try {
+      const temp_dir = resolve('./temp_backup');
+      if (existsSync(temp_dir)) {
+        await rm(temp_dir, { recursive: true, force: true });
+      }
+    } catch (cleanup_error) {
+      log.warn('Failed to clean up temporary backup files', { error: cleanup_error.message });
+    }
+    
+    throw error;
+  }
+};
+
+/**
+ * Lists all available backups from S3 storage with metadata.
+ * Retrieves backup files, sorts by modification date, and calculates total storage usage.
+ * @returns {Promise<Object>} Backup listing result
+ * @returns {Array<Object>} returns.backups - Array of backup objects
+ * @returns {string} returns.backups[].filename - Backup filename
+ * @returns {number} returns.backups[].size_bytes - File size in bytes
+ * @returns {number} returns.backups[].size_mb - File size in megabytes (rounded)
+ * @returns {string} returns.backups[].last_modified - ISO timestamp of last modification
+ * @returns {string} returns.backups[].s3_location - S3 URI of backup file
+ * @returns {number} returns.total_count - Total number of backups
+ * @returns {number} returns.total_size_bytes - Total storage used in bytes
+ * @returns {number} returns.total_size_mb - Total storage used in megabytes (rounded)
+ * @throws {Error} When S3 listing operation fails
+ */
+const list_backups = async () => {
+  const log = create_context_logger();
+  
+  try {
+    const { client, bucket } = get_s3_client();
+    
+    const command = new ListObjectsV2Command({
+      Bucket: bucket,
+      Prefix: 'joystickdb-backup-'
+    });
+    
+    const response = await client.send(command);
+    
+    if (!response.Contents || response.Contents.length === 0) {
+      return {
+        backups: [],
+        total_count: 0,
+        total_size_bytes: 0
+      };
+    }
+    
+    const backups = response.Contents
+      .filter(obj => obj.Key.startsWith('joystickdb-backup-') && obj.Key.endsWith('.tar.gz'))
+      .map(obj => ({
+        filename: obj.Key,
+        size_bytes: obj.Size,
+        size_mb: Math.round(obj.Size / 1024 / 1024),
+        last_modified: obj.LastModified.toISOString(),
+        s3_location: `s3://${bucket}/${obj.Key}`
+      }))
+      .sort((a, b) => new Date(b.last_modified) - new Date(a.last_modified));
+    
+    const total_size_bytes = backups.reduce((sum, backup) => sum + backup.size_bytes, 0);
+    
+    log.info('Listed backups', {
+      total_count: backups.length,
+      total_size_mb: Math.round(total_size_bytes / 1024 / 1024)
+    });
+    
+    return {
+      backups,
+      total_count: backups.length,
+      total_size_bytes,
+      total_size_mb: Math.round(total_size_bytes / 1024 / 1024)
+    };
+  } catch (error) {
+    log.error('Failed to list backups', { error: error.message });
+    throw error;
+  }
+};
+
+/**
+ * Restores database from a specified backup file stored in S3.
+ * Downloads backup, verifies checksum, creates safety backup of current data,
+ * extracts backup to data directory, and handles rollback on failure.
+ * @param {string} backup_filename - Name of the backup file to restore
+ * @returns {Promise<Object>} Restore operation result
+ * @returns {string} returns.backup_filename - Name of restored backup file
+ * @returns {string} returns.restored_at - ISO timestamp of restore completion
+ * @returns {number} returns.duration_ms - Time taken to restore in milliseconds
+ * @returns {string} returns.status - Restore status ('success')
+ * @throws {Error} When backup download, verification, or extraction fails
+ */
+const restore_backup = async (backup_filename) => {
+  const log = create_context_logger();
+  const restore_start = Date.now();
+  
+  try {
+    const { client, bucket } = get_s3_client();
+    
+    // NOTE: Create temporary directory.
+    const temp_dir = resolve('./temp_restore');
+    await mkdir(temp_dir, { recursive: true });
+    
+    const temp_backup_path = join(temp_dir, backup_filename);
+    
+    log.info('Starting backup restore', { backup_filename, temp_backup_path });
+    
+    // NOTE: Download backup from S3.
+    const download_command = new GetObjectCommand({
+      Bucket: bucket,
+      Key: backup_filename
+    });
+    
+    const response = await client.send(download_command);
+    
+    if (!response.Body) {
+      throw new Error('Empty backup file received from S3');
+    }
+    
+    // NOTE: Stream backup to temporary file.
+    const write_stream = createWriteStream(temp_backup_path);
+    
+    await new Promise((resolve, reject) => {
+      response.Body.pipe(write_stream);
+      write_stream.on('finish', resolve);
+      write_stream.on('error', reject);
+      response.Body.on('error', reject);
+    });
+    
+    // NOTE: Verify checksum if available in metadata.
+    if (response.Metadata && response.Metadata.checksum) {
+      const downloaded_checksum = await calculate_sha256(temp_backup_path);
+      if (downloaded_checksum !== response.Metadata.checksum) {
+        throw new Error(`Backup checksum mismatch. Expected: ${response.Metadata.checksum}, Got: ${downloaded_checksum}`);
+      }
+      log.info('Backup checksum verified', { checksum: downloaded_checksum });
+    }
+    
+    // NOTE: Stop database operations temporarily.
+    const db = get_database();
+    
+    // NOTE: Create backup of current data.
+    const current_data_backup = resolve('./data_backup_before_restore');
+    if (existsSync('./data')) {
+      const backup_current_process = spawn('cp', ['-r', './data', current_data_backup]);
+      await new Promise((resolve) => {
+        backup_current_process.on('close', resolve);
+      });
+    }
+    
+    try {
+      // NOTE: Remove current data directory.
+      if (existsSync('./data')) {
+        await rm('./data', { recursive: true, force: true });
+      }
+      
+      // NOTE: Create new data directory.
+      await mkdir('./data', { recursive: true });
+      
+      // NOTE: Extract backup.
+      const tar_process = spawn('tar', [
+        '-xzf',
+        temp_backup_path,
+        '-C',
+        './data'
+      ], {
+        stdio: ['pipe', 'pipe', 'pipe']
+      });
+      
+      let tar_error = '';
+      tar_process.stderr.on('data', (data) => {
+        tar_error += data.toString();
+      });
+      
+      const tar_exit_code = await new Promise((resolve) => {
+        tar_process.on('close', resolve);
+      });
+      
+      if (tar_exit_code !== 0) {
+        throw new Error(`Tar extraction failed with exit code ${tar_exit_code}: ${tar_error}`);
+      }
+      
+      // NOTE: Clean up temporary files.
+      unlinkSync(temp_backup_path);
+      await rm(temp_dir, { recursive: true, force: true });
+      
+      // NOTE: Clean up current data backup on success.
+      if (existsSync(current_data_backup)) {
+        await rm(current_data_backup, { recursive: true, force: true });
+      }
+      
+      const restore_duration = Date.now() - restore_start;
+      
+      log.info('Backup restore completed successfully', {
+        backup_filename,
+        duration_ms: restore_duration
+      });
+      
+      return {
+        backup_filename,
+        restored_at: new Date().toISOString(),
+        duration_ms: restore_duration,
+        status: 'success'
+      };
+    } catch (restore_error) {
+      // NOTE: Restore original data on failure.
+      if (existsSync(current_data_backup)) {
+        if (existsSync('./data')) {
+          await rm('./data', { recursive: true, force: true });
+        }
+        const restore_original_process = spawn('mv', [current_data_backup, './data']);
+        await new Promise((resolve) => {
+          restore_original_process.on('close', resolve);
+        });
+      }
+      throw restore_error;
+    }
+  } catch (error) {
+    log.error('Backup restore failed', { backup_filename, error: error.message });
+    
+    // NOTE: Clean up on failure.
+    try {
+      const temp_dir = resolve('./temp_restore');
+      if (existsSync(temp_dir)) {
+        await rm(temp_dir, { recursive: true, force: true });
+      }
+    } catch (cleanup_error) {
+      log.warn('Failed to clean up temporary restore files', { error: cleanup_error.message });
+    }
+    
+    throw error;
+  }
+};
+
+/**
+ * Cleans up old backups according to retention policy.
+ * Implements tiered retention: hourly backups for 24 hours, daily backups for 30 days,
+ * and deletes backups older than 30 days. Limits hourly backups to 24 most recent.
+ * @returns {Promise<Object>} Cleanup operation result
+ * @returns {number} returns.deleted_count - Number of backups deleted
+ * @returns {number} returns.retained_count - Number of backups retained
+ * @returns {number} returns.hourly_backups - Number of hourly backups retained
+ * @returns {number} returns.daily_backups - Number of daily backups retained
+ * @throws {Error} When backup listing or deletion operations fail
+ */
+const cleanup_old_backups = async () => {
+  const log = create_context_logger();
+  
+  try {
+    const { client, bucket } = get_s3_client();
+    
+    const backups_list = await list_backups();
+    const backups = backups_list.backups;
+    
+    if (backups.length === 0) {
+      return { deleted_count: 0, retained_count: 0 };
+    }
+    
+    // NOTE: Sort by date (newest first).
+    backups.sort((a, b) => new Date(b.last_modified) - new Date(a.last_modified));
+    
+    const now = new Date();
+    const one_hour = 60 * 60 * 1000;
+    const one_day = 24 * one_hour;
+    
+    const hourly_backups = [];
+    const daily_backups = [];
+    const to_delete = [];
+    
+    for (const backup of backups) {
+      const backup_date = new Date(backup.last_modified);
+      const age_ms = now - backup_date;
+      
+      if (age_ms <= 24 * one_hour) {
+        // NOTE: Keep hourly backups for last 24 hours.
+        hourly_backups.push(backup);
+      } else if (age_ms <= 30 * one_day) {
+        // NOTE: Keep one backup per day for last 30 days.
+        const backup_day = backup_date.toDateString();
+        const existing_daily = daily_backups.find(b => new Date(b.last_modified).toDateString() === backup_day);
+        
+        if (!existing_daily) {
+          daily_backups.push(backup);
+        } else {
+          to_delete.push(backup);
+        }
+      } else {
+        // NOTE: Delete backups older than 30 days.
+        to_delete.push(backup);
+      }
+    }
+    
+    // NOTE: Limit hourly backups to 24.
+    if (hourly_backups.length > 24) {
+      to_delete.push(...hourly_backups.slice(24));
+    }
+    
+    // NOTE: Delete old backups.
+    for (const backup of to_delete) {
+      try {
+        const delete_command = new DeleteObjectCommand({
+          Bucket: bucket,
+          Key: backup.filename
+        });
+        
+        await client.send(delete_command);
+        log.info('Deleted old backup', { filename: backup.filename });
+      } catch (delete_error) {
+        log.warn('Failed to delete backup', { 
+          filename: backup.filename, 
+          error: delete_error.message 
+        });
+      }
+    }
+    
+    const retained_count = backups.length - to_delete.length;
+    
+    log.info('Backup cleanup completed', {
+      deleted_count: to_delete.length,
+      retained_count,
+      hourly_backups: Math.min(hourly_backups.length, 24),
+      daily_backups: daily_backups.length
+    });
+    
+    return {
+      deleted_count: to_delete.length,
+      retained_count,
+      hourly_backups: Math.min(hourly_backups.length, 24),
+      daily_backups: daily_backups.length
+    };
+  } catch (error) {
+    log.error('Backup cleanup failed', { error: error.message });
+    throw error;
+  }
+};
+
+/**
+ * Starts automated backup scheduling based on configuration.
+ * Sets up periodic backup creation and cleanup according to the configured schedule
+ * (hourly, daily, or weekly). Handles invalid schedule configurations gracefully.
+ * @returns {void}
+ */
+const start_backup_schedule = () => {
+  const log = create_context_logger();
+  
+  try {
+    const settings = get_settings();
+    
+    if (!settings.s3) {
+      log.info('S3 not configured, backup scheduling disabled');
+      return;
+    }
+    
+    const schedule = settings.backup_schedule || 'hourly';
+    
+    let interval_ms;
+    switch (schedule) {
+      case 'hourly':
+        interval_ms = 60 * 60 * 1000; // 1 hour
+        break;
+      case 'daily':
+        interval_ms = 24 * 60 * 60 * 1000; // 24 hours
+        break;
+      case 'weekly':
+        interval_ms = 7 * 24 * 60 * 60 * 1000; // 7 days
+        break;
+      default:
+        log.warn('Invalid backup schedule, using hourly', { schedule });
+        interval_ms = 60 * 60 * 1000;
+    }
+    
+    if (backup_schedule_timer) {
+      clearInterval(backup_schedule_timer);
+    }
+    
+    backup_schedule_timer = setInterval(async () => {
+      try {
+        log.info('Starting scheduled backup', { schedule });
+        await create_backup();
+        await cleanup_old_backups();
+      } catch (error) {
+        log.error('Scheduled backup failed', { error: error.message });
+      }
+    }, interval_ms);
+    
+    log.info('Backup schedule started', { schedule, interval_ms });
+  } catch (error) {
+    log.error('Failed to start backup schedule', { error: error.message });
+  }
+};
+
+/**
+ * Stops the automated backup scheduling timer.
+ * Clears the active backup schedule interval and resets the timer reference.
+ * @returns {void}
+ */
+const stop_backup_schedule = () => {
+  const log = create_context_logger();
+  
+  if (backup_schedule_timer) {
+    clearInterval(backup_schedule_timer);
+    backup_schedule_timer = null;
+    log.info('Backup schedule stopped');
+  }
+};
+
+export {
+  get_s3_client,
+  test_s3_connection,
+  create_backup,
+  list_backups,
+  restore_backup,
+  cleanup_old_backups,
+  start_backup_schedule,
+  stop_backup_schedule
+};