active_postgres 0.4.0

data/lib/active_postgres/components/ssl.rb

@@ -0,0 +1,86 @@
+module ActivePostgres
+  module Components
+    class SSL < Base
+      def install
+        puts 'Installing SSL/TLS encryption...'
+
+        config.all_hosts.each do |host|
+          install_on_host(host)
+        end
+      end
+
+      def uninstall
+        puts 'SSL certificates remain (harmless to leave configured)'
+      end
+
+      def restart
+        # SSL doesn't have its own service, restart PostgreSQL
+        ssh_executor.restart_postgres(config.primary_host)
+
+        config.standby_hosts.each do |host|
+          ssh_executor.restart_postgres(host)
+        end
+      end
+
+      def install_on_standby(standby_host)
+        puts "Installing SSL on standby #{standby_host}..."
+        install_on_host(standby_host)
+      end
+
+      private
+
+      def install_on_host(host)
+        puts "  Installing SSL on #{host}..."
+
+        version = config.version
+        ssl_config = config.component_config(:ssl)
+
+        ssh_executor.ensure_postgres_user(host)
+
+        # Ensure the PostgreSQL config directory exists
+        ssh_executor.execute_on_host(host) do
+          execute :sudo, 'mkdir', '-p', "/etc/postgresql/#{version}/main"
+          execute :sudo, 'chown', 'postgres:postgres', "/etc/postgresql/#{version}/main"
+        end
+
+        ssl_cert = secrets.resolve('ssl_cert')
+        ssl_key = secrets.resolve('ssl_key')
+
+        if ssl_cert && ssl_key
+          puts '  Using SSL certificates from secrets...'
+          ssh_executor.upload_file(host, ssl_cert, "/etc/postgresql/#{version}/main/server.crt", mode: '644',
+                                                                                                 owner: 'postgres:postgres')
+          ssh_executor.upload_file(host, ssl_key, "/etc/postgresql/#{version}/main/server.key", mode: '600',
+                                                                                                owner: 'postgres:postgres')
+        else
+          puts '  Generating self-signed SSL certificates...'
+          generate_self_signed_cert(host, ssl_config)
+        end
+      end
+
+      def generate_self_signed_cert(host, ssl_config)
+        version = config.version
+        cert_path = "/etc/postgresql/#{version}/main/server.crt"
+        key_path = "/etc/postgresql/#{version}/main/server.key"
+
+        ssh_executor.execute_on_host(host) do
+          days = ssl_config[:cert_days] || 3650
+          cn = ssl_config[:common_name] || host
+
+          info "Generating self-signed certificate (CN=#{cn}, valid for #{days} days)..."
+
+          execute :sudo, 'openssl', 'req', '-new', '-x509', '-days', days.to_s,
+                  '-nodes', '-text',
+                  '-out', cert_path,
+                  '-keyout', key_path,
+                  '-subj', "/CN=#{cn}"
+
+          execute :sudo, 'chown', 'postgres:postgres', cert_path
+          execute :sudo, 'chown', 'postgres:postgres', key_path
+          execute :sudo, 'chmod', '644', cert_path
+          execute :sudo, 'chmod', '600', key_path
+        end
+      end
+    end
+  end
+end

data/lib/active_postgres/configuration.rb

@@ -0,0 +1,190 @@
+require 'yaml'
+
+module ActivePostgres
+  class Configuration
+    attr_reader :environment, :version, :user, :ssh_key, :primary, :standbys, :components, :secrets_config, :database_config
+
+    def initialize(config_hash, environment = 'development')
+      @environment = environment
+      env_config = config_hash[environment] || {}
+
+      # Check if deployment should be skipped (e.g., for development)
+      @skip_deployment = env_config['skip_deployment'] == true
+
+      @version = env_config['version'] || 18
+      @user = env_config['user'] || 'ubuntu'
+      @ssh_key = File.expand_path(env_config['ssh_key'] || '~/.ssh/id_rsa')
+
+      @primary = env_config['primary'] || {}
+      @standbys = env_config['standby'] || []
+      @standbys = [@standbys] unless @standbys.is_a?(Array)
+
+      @components = parse_components(env_config['components'] || {})
+      @secrets_config = env_config['secrets'] || {}
+    end
+
+    def self.load(config_path = 'config/postgres.yml', environment = nil)
+      environment ||= ENV['BORING_ENVIRONMENT'] || ENV['RAILS_ENV'] || 'development'
+
+      raise Error, "Config file not found: #{config_path}" unless File.exist?(config_path)
+
+      config_hash = YAML.load_file(config_path, aliases: true)
+      new(config_hash, environment)
+    end
+
+    def all_hosts
+      [primary_host] + standby_hosts
+    end
+
+    def primary_host
+      @primary['host']
+    end
+
+    def standby_hosts
+      @standbys.map { |s| s['host'] }
+    end
+
+    def component_enabled?(name)
+      @components[name]&.[](:enabled) == true
+    end
+
+    def component_config(name)
+      @components[name] || {}
+    end
+
+    def skip_deployment?
+      @skip_deployment
+    end
+
+    def primary_replication_host
+      replication_host_for(primary_host)
+    end
+
+    def replication_host_for(host)
+      node = node_config_for(host)
+      private_ip_for(node) || host
+    end
+
+    def standby_config_for(host)
+      @standbys.find { |s| s['host'] == host }
+    end
+
+    def node_label_for(host)
+      if host == primary_host
+        @primary['label']
+      else
+        standby_config_for(host)&.dig('label')
+      end
+    end
+
+    def validate!
+      raise Error, 'No primary host defined' unless primary_host
+
+      # Validate required secrets if components are enabled
+      raise Error, 'Missing replication_password secret' if component_enabled?(:repmgr) && !secrets_config['replication_password']
+
+      true
+    end
+
+    # Database and user configuration helpers from components
+    def postgres_user
+      component_config(:core)[:postgres_user] || 'postgres'
+    end
+
+    def repmgr_user
+      component_config(:repmgr)[:user] || 'repmgr'
+    end
+
+    def repmgr_database
+      component_config(:repmgr)[:database] || 'repmgr'
+    end
+
+    def pgbouncer_user
+      component_config(:pgbouncer)[:user] || 'pgbouncer'
+    end
+
+    def app_user
+      value = component_config(:core)[:app_user]
+      value.nil? || value.to_s.strip.empty? ? 'app' : value
+    end
+
+    def app_database
+      value = component_config(:core)[:app_database]
+      value.nil? || value.to_s.strip.empty? ? "app_#{environment}" : value
+    end
+
+    private
+
+    def parse_components(components_config)
+      result = {}
+
+      # Core is always enabled - include ALL core config, not just specific fields
+      core_config = components_config['core'] || {}
+      result[:core] = {
+        enabled: true,
+        version: @version,
+        locale: core_config['locale'] || 'en_US.UTF-8',
+        encoding: core_config['encoding'] || 'UTF8',
+        data_checksums: core_config['data_checksums'] != false,
+        app_user: core_config['app_user'],
+        app_database: core_config['app_database']
+      }
+
+      # Include pg_hba and postgresql config if present
+      result[:core][:pg_hba] = symbolize_keys_deep(core_config['pg_hba']) if core_config['pg_hba']
+      result[:core][:postgresql] = symbolize_keys(core_config['postgresql']) if core_config['postgresql']
+
+      # Parse each component
+      %i[repmgr pgbouncer pgbackrest monitoring ssl extensions].each do |component|
+        component_str = component.to_s
+        result[component] = if components_config[component_str]
+                              symbolize_keys(components_config[component_str])
+                            else
+                              { enabled: false }
+                            end
+      end
+
+      # Performance tuning must be explicitly enabled
+      result[:performance_tuning] = if components_config['performance_tuning']
+                                      symbolize_keys(components_config['performance_tuning'])
+                                    else
+                                      { enabled: false }
+                                    end
+
+      result
+    end
+
+    def node_config_for(host)
+      return @primary if host == primary_host
+
+      standby_config_for(host)
+    end
+
+    def private_ip_for(node_config)
+      return unless node_config
+
+      node_config['private_ip'] || node_config['host']
+    end
+
+    def symbolize_keys(hash)
+      hash.each_with_object({}) do |(key, value), result|
+        new_key = key.to_sym
+        new_value = value.is_a?(Hash) ? symbolize_keys(value) : value
+        result[new_key] = new_value
+      end
+    end
+
+    def symbolize_keys_deep(value)
+      case value
+      when Hash
+        value.each_with_object({}) do |(k, v), result|
+          result[k.to_sym] = symbolize_keys_deep(v)
+        end
+      when Array
+        value.map { |v| symbolize_keys_deep(v) }
+      else
+        value
+      end
+    end
+  end
+end

data/lib/active_postgres/connection_pooler.rb

@@ -0,0 +1,429 @@
+module ActivePostgres
+  # Production-ready connection pooling configuration for PgBouncer
+  class ConnectionPooler
+    attr_reader :config, :ssh_executor, :logger
+
+    def initialize(config, ssh_executor, logger = Logger.new)
+      @config = config
+      @ssh_executor = ssh_executor
+      @logger = logger
+    end
+
+    def setup_on_host(host)
+      @logger.info "Configuring optimized connection pooling on #{host}..."
+
+      # Analyze PostgreSQL configuration to determine optimal pool settings
+      pg_settings = get_postgresql_settings(host)
+      pool_config = calculate_pool_settings(pg_settings)
+
+      # Install PgBouncer
+      install_pgbouncer(host)
+
+      # Deploy optimized configuration
+      deploy_pgbouncer_config(host, pool_config)
+
+      # Setup authentication
+      setup_authentication(host)
+
+      # Enable and start PgBouncer
+      enable_pgbouncer(host)
+
+      # Verify the setup
+      verify_pgbouncer(host)
+    end
+
+    # Calculate optimal pool settings based on PostgreSQL max_connections
+    # This is a simpler method used by the PgBouncer component for template integration
+    def self.calculate_optimal_pool_sizes(max_connections)
+      {
+        default_pool_size: calculate_default_pool_size_static(max_connections),
+        min_pool_size: 5,
+        reserve_pool_size: 5,
+        max_client_conn: max_connections * 10,
+        max_db_connections: [max_connections - 10, 10].max,
+        max_user_connections: [max_connections - 10, 10].max
+      }
+    end
+
+    def self.calculate_default_pool_size_static(max_connections)
+      # Formula: Reserve 80% of PostgreSQL connections for the pool
+      # Split among expected number of databases/users
+      pool_per_db = (max_connections * 0.8 / 4).to_i # Assume 4 databases
+
+      # Reasonable bounds: 20-100 per pool
+      pool_per_db.clamp(20, 100)
+    end
+
+    private
+
+    def get_postgresql_settings(host)
+      settings = {}
+      postgres_user = config.postgres_user
+
+      @ssh_executor.execute_on_host(host) do
+        # Get max_connections from PostgreSQL
+        max_conn = capture(:sudo, '-u', postgres_user, 'psql', '-t', '-c',
+                           "'SHOW max_connections;'").strip.to_i
+        settings[:max_connections] = max_conn
+
+        # Get default pool mode preference
+        settings[:default_pool_mode] = 'transaction' # Best for web apps
+
+        # Get work_mem
+        work_mem = capture(:sudo, '-u', postgres_user, 'psql', '-t', '-c',
+                           "'SHOW work_mem;'").strip
+        settings[:work_mem] = work_mem
+      end
+
+      settings
+    end
+
+    def calculate_pool_settings(pg_settings)
+      max_connections = pg_settings[:max_connections]
+      postgres_user = config.postgres_user
+      pgbouncer_user = config.pgbouncer_user
+
+      # Production-optimized PgBouncer settings
+      {
+        # Connection pool settings
+        default_pool_size: calculate_default_pool_size(max_connections),
+        min_pool_size: 5,
+        reserve_pool_size: 5,
+        reserve_pool_timeout: 5,
+        max_client_conn: max_connections * 10, # Allow many client connections
+        max_db_connections: max_connections - 10, # Leave some for superuser
+        max_user_connections: max_connections - 10,
+
+        # Performance settings
+        pool_mode: pg_settings[:default_pool_mode],
+        server_reset_query: 'DISCARD ALL',
+        server_reset_query_always: 0,
+        ignore_startup_parameters: 'extra_float_digits,options',
+        disable_pqexec: 0,
+        application_name_add_host: 1,
+        conffile: '/etc/pgbouncer/pgbouncer.ini',
+        pidfile: '/var/run/pgbouncer/pgbouncer.pid',
+
+        # Connection behavior
+        server_lifetime: 3600,
+        server_idle_timeout: 600,
+        server_connect_timeout: 15,
+        server_login_retry: 15,
+        query_timeout: 0,
+        query_wait_timeout: 120,
+        client_idle_timeout: 0,
+        client_login_timeout: 60,
+        idle_transaction_timeout: 0,
+
+        # TLS/SSL settings
+        server_tls_sslmode: 'prefer',
+        server_tls_ca_file: '',
+        server_tls_key_file: '',
+        server_tls_cert_file: '',
+        server_tls_protocols: 'all',
+        server_tls_ciphers: 'fast',
+        client_tls_sslmode: 'prefer',
+        client_tls_ca_file: '',
+        client_tls_key_file: '',
+        client_tls_cert_file: '',
+        client_tls_protocols: 'all',
+        client_tls_ciphers: 'fast',
+        client_tls_ecdhcurve: 'auto',
+        client_tls_dheparams: 'auto',
+
+        # Authentication
+        auth_type: 'scram-sha-256',
+        auth_file: '/etc/pgbouncer/userlist.txt',
+        auth_hba_file: '',
+        auth_query: 'SELECT usename, passwd FROM pg_shadow WHERE usename=$1',
+        auth_user: pgbouncer_user,
+
+        # Connection sanity checks
+        server_check_delay: 30,
+        server_check_query: 'select 1',
+        server_fast_close: 0,
+        server_round_robin: 0,
+
+        # Logging
+        log_connections: 1,
+        log_disconnections: 1,
+        log_pooler_errors: 1,
+        log_stats: 1,
+        stats_period: 60,
+        verbose: 0,
+        admin_users: "#{postgres_user},#{pgbouncer_user}",
+        stats_users: "#{postgres_user},#{pgbouncer_user},stats_collector",
+
+        # Network settings
+        listen_addr: '*',
+        listen_port: 6432,
+        unix_socket_dir: '/var/run/pgbouncer',
+        unix_socket_mode: '0777',
+        unix_socket_group: '',
+
+        # Limits
+        tcp_keepalive: 1,
+        tcp_keepcnt: 9,
+        tcp_keepidle: 900,
+        tcp_keepintvl: 75,
+        tcp_user_timeout: 0,
+
+        # DNS
+        dns_max_ttl: 15,
+        dns_nxdomain_ttl: 15,
+        dns_zone_check_period: 0,
+        resolv_conf: '',
+
+        # Timeouts and limits
+        sbuf_loopcnt: 5,
+        max_packet_size: 2_147_483_647,
+        listen_backlog: 128,
+        so_reuseport: 0,
+        tcp_defer_accept: 0,
+        tcp_socket_buffer: 0,
+
+        # Process management
+        logfile: '/var/log/pgbouncer/pgbouncer.log',
+        syslog: 0,
+        syslog_ident: 'pgbouncer',
+        syslog_facility: 'daemon',
+        user: postgres_user,
+
+        # Track prepared statements per database
+        track_extra_parameters: 'IntervalStyle',
+
+        # Connection limits per user/database
+        databases: generate_database_config
+      }
+    end
+
+    def calculate_default_pool_size(max_connections)
+      # Formula: Reserve 80% of PostgreSQL connections for the pool
+      # Split among expected number of databases/users
+      pool_per_db = (max_connections * 0.8 / 4).to_i # Assume 4 databases
+
+      # Reasonable bounds: 20-100 per pool
+      pool_per_db.clamp(20, 100)
+    end
+
+    def generate_database_config
+      databases = {}
+
+      # Generate database connection strings
+      if @config.primary_host
+        databases['*'] = {
+          host: @config.primary_host,
+          port: 5432,
+          auth_user: 'pgbouncer'
+        }
+      end
+
+      # Add specific database configurations if needed
+      app_databases = @config.component_config(:pgbouncer)[:databases] || []
+      app_databases.each do |db_config|
+        databases[db_config[:name]] = {
+          host: db_config[:host] || @config.primary_host,
+          port: db_config[:port] || 5432,
+          dbname: db_config[:dbname] || db_config[:name],
+          auth_user: db_config[:auth_user] || 'pgbouncer',
+          pool_size: db_config[:pool_size] || 25,
+          reserve_pool: db_config[:reserve_pool] || 5,
+          pool_mode: db_config[:pool_mode] || 'transaction'
+        }
+      end
+
+      databases
+    end
+
+    def install_pgbouncer(host)
+      postgres_user = config.postgres_user
+
+      @ssh_executor.execute_on_host(host) do
+        # Install PgBouncer package
+        execute :sudo, 'DEBIAN_FRONTEND=noninteractive', 'apt-get', 'install',
+                '-y', '-qq', 'pgbouncer'
+
+        # Create required directories
+        execute :sudo, 'mkdir', '-p', '/var/log/pgbouncer'
+        execute :sudo, 'mkdir', '-p', '/var/run/pgbouncer'
+        execute :sudo, 'chown', '-R', "#{postgres_user}:#{postgres_user}", '/var/log/pgbouncer'
+        execute :sudo, 'chown', '-R', "#{postgres_user}:#{postgres_user}", '/var/run/pgbouncer'
+
+        # Stop default PgBouncer if running
+        begin
+          execute :sudo, 'systemctl', 'stop', 'pgbouncer'
+        rescue StandardError
+          nil
+        end
+      end
+    end
+
+    def deploy_pgbouncer_config(host, pool_config)
+      # Generate PgBouncer configuration
+      ini_content = generate_pgbouncer_ini(pool_config)
+      postgres_user = config.postgres_user
+
+      @ssh_executor.execute_on_host(host) do
+        # Upload configuration
+        upload! StringIO.new(ini_content), '/tmp/pgbouncer.ini'
+        execute :sudo, 'mv', '/tmp/pgbouncer.ini', '/etc/pgbouncer/pgbouncer.ini'
+        execute :sudo, 'chown', "#{postgres_user}:#{postgres_user}", '/etc/pgbouncer/pgbouncer.ini'
+        execute :sudo, 'chmod', '640', '/etc/pgbouncer/pgbouncer.ini'
+      end
+    end
+
+    def generate_pgbouncer_ini(settings)
+      ini = "[databases]\n"
+
+      # Add database configurations
+      settings[:databases].each do |db_name, db_config|
+        if db_name == '*'
+          # Wildcard database
+          ini += "* = host=#{db_config[:host]} port=#{db_config[:port]}"
+          ini += " auth_user=#{db_config[:auth_user]}" if db_config[:auth_user]
+        else
+          ini += "#{db_name} = "
+          ini += "host=#{db_config[:host]} "
+          ini += "port=#{db_config[:port]} "
+          ini += "dbname=#{db_config[:dbname]} " if db_config[:dbname]
+          ini += "auth_user=#{db_config[:auth_user]} " if db_config[:auth_user]
+          ini += "pool_size=#{db_config[:pool_size]} " if db_config[:pool_size]
+          ini += "reserve_pool=#{db_config[:reserve_pool]} " if db_config[:reserve_pool]
+          ini += "pool_mode=#{db_config[:pool_mode]} " if db_config[:pool_mode]
+        end
+        ini += "\n"
+      end
+
+      ini += "\n[pgbouncer]\n"
+
+      # Add PgBouncer settings
+      settings.each do |key, value|
+        next if key == :databases # Already handled above
+
+        if value.is_a?(String) && !value.empty?
+          ini += "#{key} = #{value}\n"
+        elsif value.is_a?(Integer) || value.is_a?(Float)
+          ini += "#{key} = #{value}\n"
+        end
+      end
+
+      ini
+    end
+
+    def setup_authentication(host)
+      postgres_user = config.postgres_user
+      pgbouncer_user = config.pgbouncer_user
+
+      @ssh_executor.execute_on_host(host) do
+        sql = <<~SQL.strip
+          SELECT concat('"', usename, '" "', passwd, '"')
+          FROM pg_shadow
+          WHERE passwd IS NOT NULL
+        SQL
+
+        upload! StringIO.new(sql), '/tmp/get_all_users.sql'
+        execute :chmod, '644', '/tmp/get_all_users.sql'
+        userlist = capture(:sudo, '-u', postgres_user, 'psql', '-t', '-f', '/tmp/get_all_users.sql').strip
+        execute :rm, '-f', '/tmp/get_all_users.sql'
+
+        unless userlist.include?(pgbouncer_user)
+          pgbouncer_pass = SecureRandom.hex(16)
+          escaped_pass = pgbouncer_pass.gsub("'", "''")
+
+          sql = [
+            "CREATE USER #{pgbouncer_user} WITH PASSWORD '#{escaped_pass}';",
+            "GRANT CONNECT ON DATABASE postgres TO #{pgbouncer_user};"
+          ].join("\n")
+
+          upload! StringIO.new(sql), '/tmp/create_pgbouncer_user.sql'
+          execute :chmod, '644', '/tmp/create_pgbouncer_user.sql'
+          execute :sudo, '-u', postgres_user, 'psql', '-f', '/tmp/create_pgbouncer_user.sql'
+          execute :rm, '-f', '/tmp/create_pgbouncer_user.sql'
+
+          sql = <<~SQL.strip
+            SELECT passwd
+            FROM pg_shadow
+            WHERE usename = '#{pgbouncer_user}'
+          SQL
+
+          upload! StringIO.new(sql), '/tmp/get_pgbouncer_pass.sql'
+          execute :chmod, '644', '/tmp/get_pgbouncer_pass.sql'
+          encrypted = capture(:sudo, '-u', postgres_user, 'psql', '-t', '-f', '/tmp/get_pgbouncer_pass.sql').strip
+          execute :rm, '-f', '/tmp/get_pgbouncer_pass.sql'
+
+          userlist += "\n\"#{pgbouncer_user}\" \"#{encrypted}\""
+        end
+
+        # Write userlist.txt
+        upload! StringIO.new(userlist), '/tmp/userlist.txt'
+        execute :sudo, 'mv', '/tmp/userlist.txt', '/etc/pgbouncer/userlist.txt'
+        execute :sudo, 'chown', "#{postgres_user}:#{postgres_user}", '/etc/pgbouncer/userlist.txt'
+        execute :sudo, 'chmod', '640', '/etc/pgbouncer/userlist.txt'
+      end
+    end
+
+    def enable_pgbouncer(host)
+      @ssh_executor.execute_on_host(host) do
+        # Create systemd override for running as postgres user
+        systemd_override = <<~CONF
+          [Service]
+          User=postgres
+          Group=postgres
+          RuntimeDirectory=pgbouncer
+          RuntimeDirectoryMode=0755
+
+          # Resource limits
+          LimitNOFILE=65536
+          LimitNPROC=32768
+
+          # Restart policy
+          Restart=always
+          RestartSec=10
+        CONF
+
+        upload! StringIO.new(systemd_override), '/tmp/pgbouncer_override.conf'
+        execute :sudo, 'mkdir', '-p', '/etc/systemd/system/pgbouncer.service.d/'
+        execute :sudo, 'mv', '/tmp/pgbouncer_override.conf',
+                '/etc/systemd/system/pgbouncer.service.d/override.conf'
+
+        # Reload systemd and start PgBouncer
+        execute :sudo, 'systemctl', 'daemon-reload'
+        execute :sudo, 'systemctl', 'enable', 'pgbouncer'
+        execute :sudo, 'systemctl', 'restart', 'pgbouncer'
+      end
+    end
+
+    def verify_pgbouncer(host)
+      postgres_user = config.postgres_user
+      pgbouncer_user = config.pgbouncer_user
+
+      @ssh_executor.execute_on_host(host) do
+        # Check if PgBouncer is running
+        raise "PgBouncer is not running on #{host}" unless test('systemctl is-active pgbouncer')
+
+        # Test connection through PgBouncer
+        test_conn = test('env', 'PGPASSWORD=test', 'psql', '-h', 'localhost', '-p', '6432',
+                         '-U', postgres_user, '-d', 'postgres', '-c', 'SELECT 1', '2>/dev/null')
+
+        if test_conn
+          @logger.success "✓ PgBouncer is working correctly on #{host}"
+        else
+          @logger.warn "⚠ PgBouncer is running but connection test failed on #{host}"
+        end
+
+        # Show PgBouncer stats
+        stats = begin
+          capture(:sudo, '-u', postgres_user, 'psql', '-h', 'localhost', '-p', '6432',
+                  '-U', pgbouncer_user, 'pgbouncer', '-c', 'SHOW STATS', '2>/dev/null')
+        rescue StandardError
+          nil
+        end
+
+        if stats
+          @logger.info 'PgBouncer statistics:'
+          @logger.info stats
+        end
+      end
+    end
+  end
+end