active_postgres 0.4.0

data/lib/active_postgres/performance_tuner.rb

@@ -0,0 +1,246 @@
+module ActivePostgres
+  # Automatic PostgreSQL performance tuning based on hardware specs
+  # Following best practices from PGTune and PostgreSQL documentation
+  class PerformanceTuner
+    attr_reader :config, :ssh_executor, :logger
+
+    def initialize(config, ssh_executor, logger = Logger.new)
+      @config = config
+      @ssh_executor = ssh_executor
+      @logger = logger
+    end
+
+    def tune_for_host(host, db_type: 'web')
+      @logger.info 'Analyzing hardware for optimal PostgreSQL configuration...'
+
+      hardware = analyze_hardware(host)
+      settings = calculate_optimal_settings(hardware, db_type)
+
+      @logger.info 'Hardware detected:'
+      @logger.info "  CPU Cores: #{hardware[:cpu_cores]}"
+      @logger.info "  RAM: #{format_bytes(hardware[:total_memory])}"
+      @logger.info "  Storage: #{hardware[:storage_type]}"
+
+      settings
+    end
+
+    private
+
+    def analyze_hardware(host)
+      hardware = {}
+      pg_version = @config.version.to_f # Capture before SSH block
+
+      @ssh_executor.execute_on_host(host) do
+        # Get CPU cores
+        hardware[:cpu_cores] = capture(:nproc).strip.to_i
+
+        # Get total memory in KB
+        mem_info = capture(:cat, '/proc/meminfo')
+        hardware[:total_memory] = mem_info.match(/MemTotal:\s+(\d+)/)[1].to_i * 1024
+
+        # Detect storage type (SSD vs HDD) - inline to avoid scope issues
+        disk_info = begin
+          capture(:lsblk, '-d', '-o', 'name,rota', '2>/dev/null')
+        rescue StandardError
+          ''
+        end
+        # rota=0 means SSD, rota=1 means HDD
+        hardware[:storage_type] = if disk_info.include?('nvme') || disk_info.match(/\s0$/)
+                                    'ssd'
+                                  else
+                                    'hdd'
+                                  end
+
+        # Get PostgreSQL version
+        hardware[:pg_version] = pg_version
+      end
+
+      hardware
+    end
+
+    def calculate_optimal_settings(hardware, db_type)
+      settings = {}
+
+      total_memory = hardware[:total_memory]
+      cpu_cores = hardware[:cpu_cores]
+
+      # Calculate based on database type
+      case db_type
+      when 'web'
+        # Web application (many connections, mixed read/write)
+        settings[:shared_buffers] = calculate_shared_buffers(total_memory, 0.25)
+        settings[:effective_cache_size] = calculate_effective_cache_size(total_memory, 0.75)
+        settings[:maintenance_work_mem] = calculate_maintenance_work_mem(total_memory, 0.05)
+        settings[:work_mem] = calculate_work_mem(total_memory, 200)
+        settings[:max_connections] = [200, cpu_cores * 25].min
+
+      when 'oltp'
+        # Online Transaction Processing (many small transactions)
+        settings[:shared_buffers] = calculate_shared_buffers(total_memory, 0.25)
+        settings[:effective_cache_size] = calculate_effective_cache_size(total_memory, 0.75)
+        settings[:maintenance_work_mem] = calculate_maintenance_work_mem(total_memory, 0.05)
+        settings[:work_mem] = calculate_work_mem(total_memory, 300)
+        settings[:max_connections] = [300, cpu_cores * 40].min
+
+      when 'dw'
+        # Data Warehouse (complex queries, fewer connections)
+        settings[:shared_buffers] = calculate_shared_buffers(total_memory, 0.4)
+        settings[:effective_cache_size] = calculate_effective_cache_size(total_memory, 0.8)
+        settings[:maintenance_work_mem] = calculate_maintenance_work_mem(total_memory, 0.1)
+        settings[:work_mem] = calculate_work_mem(total_memory, 50)
+        settings[:max_connections] = [50, cpu_cores * 10].min
+
+      when 'desktop'
+        # Development/Desktop (conservative settings)
+        settings[:shared_buffers] = calculate_shared_buffers(total_memory, 0.1)
+        settings[:effective_cache_size] = calculate_effective_cache_size(total_memory, 0.25)
+        settings[:maintenance_work_mem] = '64MB'
+        settings[:work_mem] = '4MB'
+        settings[:max_connections] = 20
+      end
+
+      # Common optimizations for all types
+      settings[:checkpoint_completion_target] = 0.9
+      settings[:wal_buffers] = calculate_wal_buffers(settings[:shared_buffers])
+      settings[:default_statistics_target] = 100
+      settings[:random_page_cost] = hardware[:storage_type] == 'ssd' ? 1.1 : 4
+      settings[:effective_io_concurrency] = hardware[:storage_type] == 'ssd' ? 200 : 2
+      settings[:max_worker_processes] = cpu_cores
+      settings[:max_parallel_workers_per_gather] = [(cpu_cores / 2).to_i, 4].min
+      settings[:max_parallel_workers] = cpu_cores
+      settings[:max_parallel_maintenance_workers] = [(cpu_cores / 2).to_i, 4].min
+
+      # WAL settings for replication
+      if @config.component_enabled?(:repmgr)
+        settings[:wal_level] = 'replica'
+        settings[:max_wal_senders] = [10, @config.standby_hosts.size * 2].max
+        settings[:max_replication_slots] = [10, @config.standby_hosts.size * 2].max
+        settings[:wal_keep_size] = '1GB'
+        settings[:hot_standby] = 'on'
+        settings[:wal_compression] = 'on'
+        settings[:archive_mode] = 'on'
+        settings[:archive_command] = '/bin/true' # Will be overridden by pgbackrest if enabled
+      end
+
+      # Huge pages optimization for large memory systems
+      settings[:huge_pages] = 'try' if total_memory > 32 * 1024 * 1024 * 1024 # > 32GB
+
+      # JIT compilation for PG11+
+      settings[:jit] = 'on' if hardware[:pg_version] >= 11
+
+      # Additional settings for PG13+
+      if hardware[:pg_version] >= 13
+        settings[:shared_memory_type] = hardware[:storage_type] == 'ssd' ? 'sysv' : 'mmap'
+        settings[:wal_init_zero] = 'off'
+        settings[:wal_recycle] = 'on'
+      end
+
+      # Logging optimizations
+      settings[:log_checkpoints] = 'on'
+      settings[:log_connections] = 'on'
+      settings[:log_disconnections] = 'on'
+      settings[:log_lock_waits] = 'on'
+      settings[:log_temp_files] = 0
+      settings[:log_autovacuum_min_duration] = '0'
+      settings[:log_min_duration_statement] = '1000' # Log slow queries > 1s
+
+      # Statement tracking
+      settings[:shared_preload_libraries] = 'pg_stat_statements'
+      settings['pg_stat_statements.max'] = 10_000
+      settings['pg_stat_statements.track'] = 'all'
+
+      settings
+    end
+
+    def calculate_shared_buffers(total_memory, ratio)
+      value = (total_memory * ratio).to_i
+
+      # Cap at 40% of RAM for large memory systems
+      max_value = (total_memory * 0.4).to_i
+      value = [value, max_value].min
+
+      # Minimum 128MB
+      value = [value, 128 * 1024 * 1024].max
+
+      format_memory_value(value)
+    end
+
+    def calculate_effective_cache_size(total_memory, ratio)
+      value = (total_memory * ratio).to_i
+      format_memory_value(value)
+    end
+
+    def calculate_maintenance_work_mem(total_memory, ratio)
+      value = (total_memory * ratio).to_i
+
+      # Cap at 2GB
+      max_value = 2 * 1024 * 1024 * 1024
+      value = [value, max_value].min
+
+      # Minimum 64MB
+      value = [value, 64 * 1024 * 1024].max
+
+      format_memory_value(value)
+    end
+
+    def calculate_work_mem(total_memory, max_connections)
+      # Formula: (Total RAM - shared_buffers) / (max_connections * 3)
+      shared_buffers = total_memory * 0.25
+      available_memory = total_memory - shared_buffers
+      value = (available_memory / (max_connections * 3)).to_i
+
+      # Reasonable bounds: 4MB to 256MB
+      value = value.clamp(4 * 1024 * 1024, 256 * 1024 * 1024)
+
+      format_memory_value(value)
+    end
+
+    def calculate_wal_buffers(shared_buffers)
+      # 3% of shared_buffers, capped at 16MB
+      value = parse_memory_value(shared_buffers)
+      wal_buffers = (value * 0.03).to_i
+      wal_buffers = [wal_buffers, 16 * 1024 * 1024].min
+      format_memory_value(wal_buffers)
+    end
+
+    def format_memory_value(bytes)
+      if bytes >= 1024 * 1024 * 1024
+        "#{(bytes / (1024 * 1024 * 1024)).to_i}GB"
+      elsif bytes >= 1024 * 1024
+        "#{(bytes / (1024 * 1024)).to_i}MB"
+      else
+        "#{(bytes / 1024).to_i}kB"
+      end
+    end
+
+    def parse_memory_value(value)
+      return value if value.is_a?(Integer)
+
+      match = value.match(/(\d+)\s*(GB|MB|kB|B)?/i)
+      return 0 unless match
+
+      number = match[1].to_i
+      unit = match[2]&.upcase || 'B'
+
+      case unit
+      when 'GB' then number * 1024 * 1024 * 1024
+      when 'MB' then number * 1024 * 1024
+      when 'KB' then number * 1024
+      else number
+      end
+    end
+
+    def format_bytes(bytes)
+      units = %w[B KB MB GB TB]
+      unit_index = 0
+      value = bytes.to_f
+
+      while value >= 1024 && unit_index < units.length - 1
+        value /= 1024
+        unit_index += 1
+      end
+
+      format('%.2f %s', value, units[unit_index])
+    end
+  end
+end

data/lib/active_postgres/rails/database_config.rb

@@ -0,0 +1,174 @@
+require 'yaml'
+
+module ActivePostgres
+  module Rails
+    class DatabaseConfig
+      DEFAULT_ROLE = 'primary'.freeze
+
+      def self.generate(environment = 'production', app_name: nil, config: nil)
+        builder = Builder.new(environment, app_name: app_name, config: config)
+        builder.build_connections
+      end
+
+      def self.render(environment = 'production', app_name: nil, config: nil)
+        connections = generate(environment, app_name: app_name, config: config)
+        render_yaml(environment, connections)
+      end
+
+      def self.render_yaml(environment, connections)
+        output = "#{environment}:\n"
+        connections.each do |role, config|
+          output += "  #{role}:\n"
+          output += render_hash(config, indent: 4)
+        end
+        output
+      end
+
+      def self.render_hash(hash, indent: 0)
+        output = ''
+        hash.each do |key, value|
+          spaces = ' ' * indent
+          if value.is_a?(Hash)
+            output += "#{spaces}#{key}:\n"
+            output += render_hash(value, indent: indent + 2)
+          elsif value.to_s.include?('<%=')
+            # Don't quote ERB tags
+            output += "#{spaces}#{key}: #{value}\n"
+          elsif value.is_a?(String)
+            output += "#{spaces}#{key}: #{value}\n"
+          elsif [true, false].include?(value)
+            output += "#{spaces}#{key}: #{value}\n"
+          elsif value.is_a?(Integer)
+            output += "#{spaces}#{key}: #{value}\n"
+          else
+            output += "#{spaces}#{key}: #{value}\n"
+          end
+        end
+        output
+      end
+
+      def self.render_partial(environment = 'production', app_name: nil, config: nil)
+        <<~YAML
+          # Generated by active_postgres. This file is evaluated via ERB from config/database.yml.
+          # Update config/postgres.yml or the POSTGRES_* environment variables to change values.
+
+          #{render(environment, app_name: app_name, config: config)}
+        YAML
+      end
+
+      class Builder
+        attr_reader :environment, :config, :app_name
+
+        def initialize(environment, app_name: nil, config: nil)
+          @environment = environment
+          @app_name = normalize_app_name(app_name)
+          @config = config || Configuration.load('config/postgres.yml', environment)
+        end
+
+        def build_connections
+          connections = {
+            'primary' => deep_merge(base_connection, primary_connection_overrides)
+          }
+
+          replica_config = replica_connection_overrides
+          connections['primary_replica'] = deep_merge(base_connection, replica_config) if replica_config
+
+          connections
+        end
+
+        private
+
+        def base_connection
+          {
+            'adapter' => 'postgresql',
+            'encoding' => 'unicode',
+            'pool' => env_value('RAILS_MAX_THREADS', fallback: 5),
+            'username' => env_value('POSTGRES_APP_USER', fallback: app_name),
+            'password' => env_value('POSTGRES_APP_PASSWORD'),
+            'variables' => {
+              'statement_timeout' => env_value('POSTGRES_STATEMENT_TIMEOUT', fallback: '15s')
+            }
+          }
+        end
+
+        def primary_connection_overrides
+          primary_config = {
+            'database' => env_value('POSTGRES_DATABASE', fallback: default_database_name),
+            'host' => env_value('POSTGRES_PRIMARY_HOST', fallback: config.primary_replication_host),
+            'port' => env_value('POSTGRES_PRIMARY_PORT', fallback: 5432)
+          }
+
+          # Set application_name in variables, not at top level
+          primary_config['variables'] = (primary_config['variables'] || {}).merge(
+            'application_name' => env_value('POSTGRES_APPLICATION_NAME', fallback: "#{app_name}-primary")
+          )
+
+          primary_config
+        end
+
+        def replica_connection_overrides
+          standby_host = config.standby_hosts.first
+          return nil unless standby_host
+
+          resolved_host = config.replication_host_for(standby_host)
+
+          replica_config = {
+            'database' => env_value('POSTGRES_DATABASE', fallback: default_database_name),
+            'host' => env_value('POSTGRES_REPLICA_HOST', fallback: resolved_host),
+            'port' => env_value('POSTGRES_REPLICA_PORT', fallback: 5432),
+            'replica' => true,
+            'role' => 'reading'
+          }
+
+          # Set application_name in variables, not at top level
+          replica_config['variables'] = (replica_config['variables'] || {}).merge(
+            'application_name' => env_value('POSTGRES_APPLICATION_NAME', fallback: "#{app_name}-replica")
+          )
+
+          replica_config
+        end
+
+        def env_value(name, fallback: nil)
+          if fallback.nil?
+            "<%= ENV['#{name}'] %>"
+          elsif fallback.is_a?(String)
+            "<%= ENV.fetch('#{name}') { '#{fallback}' } %>"
+          else
+            "<%= ENV.fetch('#{name}') { #{fallback} } %>"
+          end
+        end
+
+        def normalize_app_name(custom_name)
+          chosen = custom_name || ENV['BORING_APP_NAME'] || default_app_name
+          chosen.to_s.tr('- ', '_')
+        end
+
+        def default_app_name
+          if defined?(::Rails) && ::Rails.application
+            ::Rails.application.class.module_parent_name.underscore
+          else
+            File.basename(Dir.pwd).tr('- ', '_')
+          end
+        rescue StandardError
+          'app'
+        end
+
+        def default_database_name
+          "#{app_name}_#{environment}"
+        end
+
+        def deep_merge(hash1, hash2)
+          result = hash1.dup
+          hash2.each do |key, value|
+            result[key] = if result[key].is_a?(Hash) && value.is_a?(Hash)
+                            deep_merge(result[key], value)
+                          else
+                            value
+                          end
+          end
+          result
+        end
+      end
+    end
+  end
+end

data/lib/active_postgres/rails/migration_guard.rb

@@ -0,0 +1,25 @@
+module ActivePostgres
+  module Rails
+    module MigrationGuard
+      def exec_migration(conn, direction)
+        # Check if we're connected to a read replica
+        if connection_is_replica?(conn)
+          raise ActiveRecord::MigrationError,
+                'Cannot run migrations on read replica! Connect to primary database.'
+        end
+
+        super
+      end
+
+      private
+
+      def connection_is_replica?(conn)
+        # Check if PostgreSQL is in recovery mode (i.e., it's a replica)
+        result = conn.execute('SELECT pg_is_in_recovery();')
+        result.first['pg_is_in_recovery'] == 't'
+      rescue StandardError
+        false
+      end
+    end
+  end
+end

data/lib/active_postgres/railtie.rb

@@ -0,0 +1,28 @@
+if defined?(Rails::Railtie)
+  require 'rails/railtie'
+
+  module ActivePostgres
+    class Railtie < ::Rails::Railtie
+      railtie_name :active_postgres
+
+      rake_tasks do
+        Dir[File.expand_path('../tasks/**/*.rake', __dir__)].each { |f| load f }
+      end
+
+      generators do
+        require_relative 'generators/active_postgres/install_generator'
+      end
+
+      initializer 'active_postgres.migration_guard' do
+        ::ActiveSupport.on_load(:active_record) do
+          require_relative 'rails/migration_guard'
+          ::ActiveRecord::Migration.prepend(ActivePostgres::Rails::MigrationGuard)
+        end
+      end
+
+      console do
+        puts 'ActivePostgres loaded. Try: ActivePostgres.status'
+      end
+    end
+  end
+end

data/lib/active_postgres/retry_helper.rb

@@ -0,0 +1,80 @@
+module ActivePostgres
+  module RetryHelper
+    class RetryExhausted < StandardError; end
+
+    # Retry a block with exponential backoff
+    # @param max_attempts [Integer] Maximum number of attempts
+    # @param initial_delay [Float] Initial delay in seconds
+    # @param max_delay [Float] Maximum delay between retries
+    # @param backoff_factor [Float] Multiplier for exponential backoff
+    # @param on [Array<Class>] Exception classes to retry on
+    # @yield Block to retry
+    # @return Result of the block
+    def retry_with_backoff(max_attempts: 3, initial_delay: 1.0, max_delay: 30.0,
+                           backoff_factor: 2.0, on: [StandardError])
+      attempt = 0
+      delay = initial_delay
+
+      begin
+        attempt += 1
+        yield
+      rescue *on => e
+        if attempt < max_attempts
+          puts "  Attempt #{attempt}/#{max_attempts} failed: #{e.message}"
+          puts "  Retrying in #{delay.round(1)}s..."
+          sleep delay
+          delay = [delay * backoff_factor, max_delay].min
+          retry
+        else
+          puts "  All #{max_attempts} attempts failed"
+          raise RetryExhausted, "Failed after #{max_attempts} attempts: #{e.message}"
+        end
+      end
+    end
+
+    # Wait for a condition to be true with timeout
+    # @param timeout [Float] Maximum time to wait in seconds
+    # @param interval [Float] Time between checks in seconds
+    # @param description [String] Description of what we're waiting for
+    # @yield Block that should return true when condition is met
+    # @return [Boolean] true if condition met, false if timeout
+    def wait_for(timeout: 60, interval: 3, description: 'condition')
+      deadline = Time.now + timeout
+      attempts = 0
+
+      while Time.now < deadline
+        attempts += 1
+
+        begin
+          return true if yield
+        rescue StandardError => e
+          puts "  Check #{attempts} raised error: #{e.message}" if (attempts % 10).zero?
+        end
+
+        remaining = (deadline - Time.now).to_i
+        puts "  Waiting for #{description}... (#{remaining}s remaining)" if (attempts % 5).zero?
+        sleep interval
+      end
+
+      puts "  ⚠️  Timeout waiting for #{description} after #{timeout}s"
+      false
+    end
+
+    # Execute a block with a timeout
+    # @param timeout [Float] Timeout in seconds
+    # @param description [String] Description of the operation
+    # @yield Block to execute
+    # @return Result of the block
+    def with_timeout(timeout: 300, description: 'operation')
+      result = nil
+      thread = Thread.new { result = yield }
+
+      unless thread.join(timeout)
+        thread.kill
+        raise Timeout::Error, "#{description} timed out after #{timeout}s"
+      end
+
+      result
+    end
+  end
+end

data/lib/active_postgres/rollback_manager.rb

@@ -0,0 +1,140 @@
+module ActivePostgres
+  class RollbackManager
+    attr_reader :config, :ssh_executor, :logger, :rollback_stack
+
+    def initialize(config, ssh_executor, logger: nil)
+      @config = config
+      @ssh_executor = ssh_executor
+      @logger = logger || Logger.new
+      @rollback_stack = []
+    end
+
+    # Register a rollback action
+    # @param description [String] Description of what will be rolled back
+    # @param host [String] Host to execute rollback on
+    # @yield Block to execute for rollback
+    def register(description, host: nil, &block)
+      rollback_stack.push({
+                            description: description,
+                            host: host,
+                            action: block
+                          })
+    end
+
+    # Execute all registered rollback actions in reverse order
+    def execute
+      return if rollback_stack.empty?
+
+      logger.warn "Executing rollback (#{rollback_stack.count} actions)..."
+
+      rollback_stack.reverse.each do |rollback|
+        logger.info "  Rolling back: #{rollback[:description]}"
+
+        if rollback[:host]
+          ssh_executor.execute_on_host(rollback[:host]) do
+            instance_eval(&rollback[:action])
+          end
+        else
+          rollback[:action].call
+        end
+
+        logger.success '    Completed'
+      rescue StandardError => e
+        logger.error "    Failed: #{e.message}"
+        # Continue with other rollback actions
+      end
+
+      clear
+      logger.success 'Rollback completed'
+    end
+
+    # Clear all registered rollback actions
+    def clear
+      rollback_stack.clear
+    end
+
+    # Wrap a block with automatic rollback on failure
+    # @param description [String] Description of the operation
+    # @yield Block to execute
+    def with_rollback(description: 'operation')
+      result = yield
+      clear # Success - clear rollback stack
+      result
+    rescue StandardError => e
+      logger.error "#{description} failed: #{e.message}"
+      execute if rollback_stack.any?
+      raise
+    end
+
+    # Common rollback actions for PostgreSQL components
+
+    def register_postgres_cluster_removal(host, version)
+      register("Remove PostgreSQL cluster on #{host}", host: host) do
+        begin
+          execute :sudo, 'systemctl', 'stop', 'postgresql'
+        rescue StandardError
+          nil
+        end
+        execute :sudo, 'pg_dropcluster', '--stop', version.to_s, 'main', rescue: nil
+      end
+    end
+
+    def register_package_removal(host, packages)
+      register("Remove packages on #{host}: #{packages.join(', ')}", host: host) do
+        execute :sudo, 'apt-get', 'remove', '-y', *packages
+      rescue StandardError
+        nil
+      end
+    end
+
+    def register_file_removal(host, file_path)
+      register("Remove file #{file_path} on #{host}", host: host) do
+        execute :sudo, 'rm', '-f', file_path
+      rescue StandardError
+        nil
+      end
+    end
+
+    def register_directory_removal(host, dir_path)
+      register("Remove directory #{dir_path} on #{host}", host: host) do
+        execute :sudo, 'rm', '-rf', dir_path
+      rescue StandardError
+        nil
+      end
+    end
+
+    def register_user_removal(host, username)
+      register("Remove user #{username} on #{host}", host: host) do
+        execute :sudo, 'userdel', username
+      rescue StandardError
+        nil
+      end
+    end
+
+    def register_database_removal(host, database_name)
+      postgres_user = config.postgres_user
+      register("Drop database #{database_name} on #{host}", host: host) do
+        sql = "DROP DATABASE IF EXISTS #{database_name};"
+        upload! StringIO.new(sql), '/tmp/drop_database.sql'
+        execute :chmod, '644', '/tmp/drop_database.sql'
+        execute :sudo, '-u', postgres_user, 'psql', '-f', '/tmp/drop_database.sql'
+        execute :rm, '-f', '/tmp/drop_database.sql'
+      rescue StandardError
+        nil
+      end
+    end
+
+    def register_postgres_user_removal(host, username)
+      postgres_user = config.postgres_user
+      register("Drop PostgreSQL user #{username} on #{host}", host: host) do
+        sql = "DROP USER IF EXISTS #{username};"
+        upload! StringIO.new(sql), '/tmp/drop_user.sql'
+        execute :chmod, '644', '/tmp/drop_user.sql'
+        execute :sudo, '-u', postgres_user, 'psql', '-f', '/tmp/drop_user.sql'
+        execute :rm, '-f', '/tmp/drop_user.sql'
+      rescue StandardError
+        nil
+      end
+    end
+  end
+end