active_postgres 0.4.0

data/lib/active_postgres/secrets.rb

@@ -0,0 +1,86 @@
+require 'English'
+module ActivePostgres
+  class Secrets
+    attr_reader :config
+
+    def initialize(config)
+      @config = config
+      @cache = {}
+    end
+
+    def resolve(secret_key)
+      return @cache[secret_key] if @cache.key?(secret_key)
+
+      secret_value = config.secrets_config[secret_key]
+      return nil unless secret_value
+
+      resolved = resolve_secret_value(secret_value)
+      @cache[secret_key] = resolved
+      resolved
+    end
+
+    def resolve_all
+      config.secrets_config.keys.each_with_object({}) do |key, result|
+        result[key] = resolve(key)
+      end
+    end
+
+    def cache_to_files(directory = '.secrets')
+      require 'fileutils'
+
+      FileUtils.mkdir_p(directory)
+
+      resolve_all.each do |key, value|
+        file_path = File.join(directory, key)
+        File.write(file_path, value)
+        File.chmod(0o600, file_path)
+        puts "Cached #{key} to #{file_path}"
+      end
+
+      puts "\n✓ Secrets cached to #{directory}/"
+      puts "Add to .gitignore: #{directory}/"
+    end
+
+    private
+
+    def resolve_secret_value(value)
+      case value
+      when /^rails_credentials:(.+)$/
+        # Rails credentials: rails_credentials:postgres.superuser_password
+        key_path = ::Regexp.last_match(1)
+        fetch_from_rails_credentials(key_path)
+      when /^\$\((.+)\)$/
+        # Command execution: $(op read "op://...")
+        execute_command(::Regexp.last_match(1))
+      when /^\$([A-Z_][A-Z0-9_]*)$/
+        # Environment variable: $POSTGRES_PASSWORD
+        ENV.fetch(::Regexp.last_match(1), nil)
+      when /^env:(.+)$/
+        # Explicit env var: env:DATABASE_PASSWORD
+        ENV.fetch(::Regexp.last_match(1), nil)
+      else
+        # Literal value
+        value
+      end
+    end
+
+    def fetch_from_rails_credentials(key_path)
+      return nil unless Credentials.available?
+
+      keys = key_path.split('.').map(&:to_sym)
+      Rails.application.credentials.dig(*keys)
+    end
+
+    def execute_command(command)
+      # Preserve RAILS_ENV if set
+      env_prefix = ENV['RAILS_ENV'] ? "RAILS_ENV=#{ENV['RAILS_ENV']} " : ''
+      full_command = "#{env_prefix}#{command}"
+
+      result = `#{full_command}`.strip
+
+      raise Error, "Failed to execute secret command: #{command} (exit status: #{$CHILD_STATUS.exitstatus})" unless $CHILD_STATUS.success?
+
+      result
+    end
+  end
+end

data/lib/active_postgres/ssh_executor.rb

@@ -0,0 +1,288 @@
+require 'sshkit'
+require 'sshkit/dsl'
+require 'securerandom'
+
+module ActivePostgres
+  class SSHExecutor
+    include SSHKit::DSL
+
+    attr_reader :config
+
+    def initialize(config, quiet: false)
+      @config = config
+      @quiet = quiet
+      setup_sshkit
+    end
+
+    def quiet?
+      @quiet
+    end
+
+    def execute_on_host(host, &)
+      on("#{config.user}@#{host}", &)
+    end
+
+    def execute_on_primary(&)
+      execute_on_host(config.primary_host, &)
+    end
+
+    def execute_on_standbys(&)
+      hosts = config.standby_hosts.map { |h| "#{config.user}@#{h}" }
+      on(hosts, in: :parallel, &)
+    end
+
+    def execute_on_all_hosts(&)
+      hosts = config.all_hosts.map { |h| "#{config.user}@#{h}" }
+      on(hosts, in: :parallel, &)
+    end
+
+    def install_postgres(host, version = 18)
+      execute_on_host(host) do
+        info "Installing PostgreSQL #{version}..."
+
+        if test('[ -f /etc/apt/sources.list.d/pgdg.list ]')
+          execute :sudo, 'rm', '-f',
+                  '/etc/apt/sources.list.d/pgdg.list'
+        end
+
+        execute :sudo, 'apt-get', 'update', '-qq'
+        execute :sudo, 'DEBIAN_FRONTEND=noninteractive', 'apt-get', 'install', '-y', '-qq', 'gnupg', 'wget',
+                'lsb-release', 'locales'
+
+        info 'Generating locales...'
+        execute :sudo, 'locale-gen', 'en_US.UTF-8'
+        execute :sudo, 'update-locale', 'LANG=en_US.UTF-8'
+
+        # Check for any installed PostgreSQL server packages
+        if test('command -v pg_lsclusters')
+          existing_clusters = capture(:pg_lsclusters, '-h').split("\n")
+          installed_versions = existing_clusters.map { |line| line.split[0].to_i }.uniq.sort
+
+          # Check if we have different versions installed
+          other_versions = installed_versions - [version]
+
+          if other_versions.any?
+            info "Found PostgreSQL version(s) #{other_versions.join(', ')}, cleaning up for fresh PostgreSQL #{version} install..."
+
+            # Stop all PostgreSQL services
+            execute :sudo, 'systemctl', 'stop', 'postgresql' if test('systemctl is-active postgresql')
+
+            # Remove all PostgreSQL packages
+            execute :sudo, 'DEBIAN_FRONTEND=noninteractive', 'apt-get', 'remove', '--purge', '-y', '-qq', 'postgresql*'
+            execute :sudo, 'apt-get', 'autoremove', '-y', '-qq'
+
+            # Clean up OLD version directories only (preserve target version SSL certs, etc.)
+            other_versions.each do |old_version|
+              execute :sudo, 'rm', '-rf', "/etc/postgresql/#{old_version}"
+              execute :sudo, 'rm', '-rf', "/var/lib/postgresql/#{old_version}"
+            end
+
+            execute :sudo, 'rm', '-f', '/etc/apt/sources.list.d/pgdg.list'
+            execute :sudo, 'rm', '-f', '/usr/share/keyrings/postgresql-archive-keyring.gpg'
+
+            info 'Cleanup complete'
+          elsif installed_versions.include?(version)
+            info "PostgreSQL #{version} already installed, skipping cleanup"
+          end
+        end
+
+        info 'Ensuring PostgreSQL GPG key is present...'
+        execute :wget, '--quiet', '-O', '/tmp/pgdg.asc', 'https://www.postgresql.org/media/keys/ACCC4CF8.asc'
+        execute :sudo, 'gpg', '--dearmor', '--yes', '-o', '/usr/share/keyrings/postgresql-archive-keyring.gpg',
+                '/tmp/pgdg.asc'
+        execute :rm, '/tmp/pgdg.asc'
+
+        info 'Configuring PostgreSQL apt repository...'
+        pgdg_repo = "'echo \"deb [signed-by=/usr/share/keyrings/postgresql-archive-keyring.gpg] " \
+                    'http://apt.postgresql.org/pub/repos/apt $(lsb_release -cs)-pgdg main" > ' \
+                    "/etc/apt/sources.list.d/pgdg.list'"
+        execute :sudo, 'sh', '-c', pgdg_repo
+
+        execute :sudo, 'apt-get', 'update', '-qq'
+        execute :sudo, 'DEBIAN_FRONTEND=noninteractive', 'apt-get', 'install', '-y', '-qq', "postgresql-#{version}",
+                "postgresql-client-#{version}"
+
+        execute :sudo, 'systemctl', 'enable', 'postgresql'
+        execute :sudo, 'systemctl', 'start', 'postgresql' unless test('systemctl is-active postgresql')
+      end
+    end
+
+    def upload_file(host, content, remote_path, mode: '644', owner: nil)
+      execute_on_host(host) do
+        temp_file = "/tmp/#{File.basename(remote_path)}"
+        upload! StringIO.new(content), temp_file
+
+        execute :sudo, 'mv', temp_file, remote_path
+        execute :sudo, 'chown', owner, remote_path if owner
+        execute :sudo, 'chmod', mode, remote_path
+      end
+    end
+
+    def ensure_postgres_user(host)
+      postgres_user = config.postgres_user
+
+      execute_on_host(host) do
+        execute :sudo, 'groupadd', '--system', postgres_user unless test(:getent, 'group', postgres_user)
+
+        unless test(:id, postgres_user)
+          execute :sudo, 'useradd', '--system', '--home', '/var/lib/postgresql',
+                  '--shell', '/bin/bash', '--gid', postgres_user, '--create-home', postgres_user
+        end
+      end
+    end
+
+    def postgres_running?(host)
+      result = false
+      execute_on_host(host) do
+        # Check if any PostgreSQL cluster is online using pg_lsclusters
+        # This works regardless of whether it's the generic postgresql service
+        # or a specific postgresql@version-main service
+        clusters = begin
+          capture(:sudo, 'pg_lsclusters', '2>/dev/null')
+        rescue StandardError
+          ''
+        end
+        result = clusters.include?('online')
+      end
+      result
+    end
+
+    def restart_postgres(host, version = nil)
+      execute_on_host(host) do
+        if version
+          begin
+            execute :sudo, 'pg_ctlcluster', version.to_s, 'main', 'restart'
+          rescue StandardError => e
+            error "Failed to restart PostgreSQL cluster #{version}/main"
+            info 'Checking systemd logs...'
+            logs = begin
+              capture(:sudo, 'journalctl', '-xeu', "postgresql@#{version}-main", '-n', '50',
+                      '--no-pager')
+            rescue StandardError
+              'Could not get systemd logs'
+            end
+            info logs
+            info 'Checking PostgreSQL logs...'
+            pg_logs = begin
+              capture(:sudo, 'tail', '-100',
+                      "/var/log/postgresql/postgresql-#{version}-main.log")
+            rescue StandardError
+              'Could not get PostgreSQL logs'
+            end
+            info pg_logs
+            info 'Checking cluster status...'
+            cluster_status = begin
+              capture(:sudo, 'pg_lsclusters')
+            rescue StandardError
+              'Could not get cluster status'
+            end
+            info cluster_status
+            raise e
+          end
+        else
+          execute :sudo, 'systemctl', 'restart', 'postgresql'
+        end
+      end
+    end
+
+    def stop_postgres(host)
+      execute_on_host(host) do
+        execute :sudo, 'systemctl', 'stop', 'postgresql'
+      end
+    end
+
+    def get_postgres_status(host)
+      result = nil
+      postgres_user = config.postgres_user
+      execute_on_host(host) do
+        result = capture(:sudo, '-u', postgres_user, 'psql', '-c', 'SELECT version();')
+      end
+      result
+    end
+
+    def run_sql(host, sql)
+      result = nil
+      postgres_user = config.postgres_user
+      execute_on_host(host) do
+        # Use a temporary file to avoid shell escaping issues with special characters
+        temp_file = "/tmp/query_#{SecureRandom.hex(8)}.sql"
+        upload! StringIO.new(sql), temp_file
+        execute :chmod, '644', temp_file
+
+        begin
+          result = capture(:sudo, '-u', postgres_user, 'psql', '-t', '-f', temp_file)
+        ensure
+          execute :rm, '-f', temp_file
+        end
+      end
+      result
+    end
+
+    def ensure_cluster_exists(host, version)
+      execute_on_host(host) do
+        data_dir = "/var/lib/postgresql/#{version}/main"
+
+        if test(:sudo, 'test', '-d', data_dir)
+          info "PostgreSQL #{version}/main cluster already exists, skipping creation"
+        else
+          info 'Creating PostgreSQL cluster...'
+          execute :sudo, 'pg_createcluster', version.to_s, 'main', '--start'
+        end
+      end
+    end
+
+    def recreate_cluster(host, version)
+      execute_on_host(host) do
+        info 'Ensuring clean cluster state...'
+        begin
+          execute :sudo, 'systemctl', 'stop', 'postgresql'
+        rescue StandardError
+          nil
+        end
+        begin
+          execute :sudo, 'pg_dropcluster', '--stop', version.to_s, 'main'
+        rescue StandardError
+          nil
+        end
+        begin
+          execute :sudo, 'rm', '-rf', "/etc/postgresql/#{version}/main"
+        rescue StandardError
+          nil
+        end
+        begin
+          execute :sudo, 'rm', '-rf', "/var/lib/postgresql/#{version}/main"
+        rescue StandardError
+          nil
+        end
+
+        info 'Creating fresh PostgreSQL cluster...'
+        execute :sudo, 'pg_createcluster', version.to_s, 'main'
+        execute :sudo, 'systemctl', 'start', 'postgresql'
+      end
+    end
+
+    private
+
+    def setup_sshkit
+      if @quiet
+        SSHKit.config.output_verbosity = ::Logger::FATAL
+        SSHKit.config.format = :blackhole
+      else
+        SSHKit.config.output_verbosity = ::Logger::INFO
+        SSHKit.config.format = :pretty
+      end
+
+      return unless File.exist?(config.ssh_key)
+
+      SSHKit::Backend::Netssh.configure do |ssh|
+        ssh.ssh_options = {
+          keys: [config.ssh_key],
+          keys_only: true,
+          forward_agent: false,
+          auth_methods: ['publickey'],
+          verify_host_key: :never
+        }
+      end
+    end
+  end
+end

data/lib/active_postgres/standby_deployment_flow.rb

@@ -0,0 +1,122 @@
+module ActivePostgres
+  class StandbyDeploymentFlow < DeploymentFlow
+    attr_reader :standby_host
+
+    def initialize(config, standby_host:, **)
+      super(config, **)
+      @standby_host = standby_host
+    end
+
+    private
+
+    def operation_name
+      "Standby Setup: #{standby_host}"
+    end
+
+    def print_targets
+      logger.info "Primary: #{config.primary_host}"
+      logger.info "Standby: #{standby_host}"
+    end
+
+    def validate_specific_requirements
+      abort "❌ #{standby_host} is not configured as a standby in config/postgres.yml" unless config.standby_hosts.include?(standby_host)
+
+      return if config.component_enabled?(:repmgr)
+
+      abort '❌ repmgr component must be enabled to setup standbys'
+    end
+
+    def run_preflight_checks
+      super
+
+      check_if_standby_already_deployed
+    end
+
+    def check_if_standby_already_deployed
+      logger.info "\nChecking if standby is already deployed..."
+
+      postgres_running = ssh_executor.postgres_running?(standby_host)
+
+      return unless postgres_running
+
+      logger.warn "⚠️  PostgreSQL is already running on #{standby_host}"
+      logger.warn '   This deployment will DROP and RECREATE the database cluster'
+      logger.warn '   All data on this standby will be LOST and re-cloned from primary'
+      puts ''
+    rescue StandardError => e
+      logger.warn "Could not check if standby is deployed: #{e.message}"
+    end
+
+    def list_deployment_steps
+      logger.info "  • Install PostgreSQL #{config.version} packages on #{standby_host}"
+      logger.info "  • Clone data from primary: #{config.primary_host}"
+      logger.info '  • Register standby with repmgr cluster'
+    end
+
+    def list_warnings
+      logger.info "\n⚠️  Primary database will NOT be touched"
+    end
+
+    def deploy_components
+      deploy_ssl if config.component_enabled?(:ssl)
+      deploy_core
+      deploy_repmgr
+      deploy_optional_components
+    end
+
+    def deploy_ssl
+      logger.task('Setting up SSL on standby') do
+        component = Components::SSL.new(config, ssh_executor, secrets)
+        register_rollback('SSL', component)
+        component.install_on_standby(standby_host)
+      end
+    end
+
+    def deploy_core
+      logger.task('Installing PostgreSQL packages on standby') do
+        component = Components::Core.new(config, ssh_executor, secrets)
+        component.install_packages_only(standby_host)
+      end
+    end
+
+    def deploy_repmgr
+      logger.task('Setting up repmgr and cloning from primary') do
+        component = Components::Repmgr.new(config, ssh_executor, secrets)
+        register_rollback('repmgr', component)
+        component.setup_standby_only(standby_host)
+      end
+    end
+
+    def deploy_optional_components
+      deploy_pgbouncer if config.component_enabled?(:pgbouncer)
+      deploy_monitoring if config.component_enabled?(:monitoring)
+    end
+
+    def deploy_pgbouncer
+      logger.task('Setting up pgbouncer on standby') do
+        component = Components::PgBouncer.new(config, ssh_executor, secrets)
+        component.install_on_standby(standby_host) if component.respond_to?(:install_on_standby)
+      end
+    end
+
+    def deploy_monitoring
+      logger.task('Setting up monitoring on standby') do
+        component = Components::Monitoring.new(config, ssh_executor, secrets)
+        component.install_on_standby(standby_host) if component.respond_to?(:install_on_standby)
+      end
+    end
+
+    def register_rollback(component_name, component)
+      rollback_manager.register("Uninstall #{component_name} on #{standby_host}", host: standby_host) do
+        component.uninstall
+      rescue StandardError => e
+        logger.warn "Failed to uninstall #{component_name} on #{standby_host}: #{e.message}"
+      end
+    end
+
+    def list_next_steps
+      logger.info '  1. Check cluster status: active_postgres status'
+      logger.info '  2. Verify replication: Check repmgr cluster show'
+    end
+  end
+end

data/lib/active_postgres/validator.rb

@@ -0,0 +1,143 @@
+module ActivePostgres
+  class Validator
+    attr_reader :config, :ssh_executor, :errors, :warnings
+
+    def initialize(config, ssh_executor)
+      @config = config
+      @ssh_executor = ssh_executor
+      @errors = []
+      @warnings = []
+    end
+
+    # Run all validation checks before installation
+    def validate_all
+      puts 'Running pre-flight validation checks...'
+
+      validate_configuration
+      validate_ssh_connectivity
+      validate_network_connectivity
+      validate_system_requirements
+
+      if errors.any?
+        puts "\n❌ Validation failed with #{errors.count} error(s):"
+        errors.each { |error| puts "  - #{error}" }
+        return false
+      end
+
+      if warnings.any?
+        puts "\n⚠️  Found #{warnings.count} warning(s):"
+        warnings.each { |warning| puts "  - #{warning}" }
+      end
+
+      puts "✅ All validation checks passed!\n"
+      true
+    end
+
+    private
+
+    def validate_configuration
+      puts '  Checking configuration...'
+
+      # Validate primary host
+      errors << 'Primary host not configured' unless config.primary_host
+
+      # Validate PostgreSQL version
+      errors << "PostgreSQL version must be 12 or higher (got: #{config.version})" unless config.version && config.version >= 12
+
+      # Validate repmgr setup
+      if config.component_enabled?(:repmgr)
+        errors << 'repmgr enabled but no standby hosts configured' unless config.standby_hosts&.any?
+
+        replication_host = config.primary_replication_host
+        if replication_host == config.primary_host
+          warnings << "Primary is using '#{config.primary_host}' for replication traffic. Set private_ip for isolated networks."
+        end
+
+        config.standby_hosts.each do |host|
+          standby_replication_host = config.replication_host_for(host)
+          next unless standby_replication_host == host
+
+          warnings << "Standby #{host} is using its SSH host for replication. Provide private_ip if it differs."
+        end
+      end
+
+      # Validate SSL configuration
+      return unless config.component_enabled?(:ssl)
+
+      ssl_config = config.component_config(:ssl)
+      return unless ssl_config['certificate_mode'] == 'custom'
+
+      warnings << 'Custom SSL certificates configured - ensure they are available'
+    end
+
+    def validate_ssh_connectivity
+      puts '  Checking SSH connectivity...'
+
+      all_hosts = [config.primary_host] + config.standby_hosts
+
+      all_hosts.each do |host|
+        ssh_executor.execute_on_host(host) do
+          test(:echo, 'SSH connection test')
+        end
+      rescue StandardError => e
+        errors << "Cannot connect to #{host} via SSH: #{e.message}"
+      end
+    end
+
+    def validate_network_connectivity
+      puts '  Checking network connectivity...'
+
+      return unless config.component_enabled?(:repmgr)
+
+      primary_replication_host = config.primary_replication_host
+      return unless primary_replication_host
+
+      # Capture errors and warnings before entering SSH block
+      validator_errors = errors
+      validator_warnings = warnings
+
+      # Check if standbys can reach primary via the preferred replication network
+      config.standby_hosts.each do |host|
+        ssh_executor.execute_on_host(host) do
+          can_ping = test(:ping, '-c', '1', '-W', '2', primary_replication_host)
+          validator_errors << "Standby #{host} cannot reach the primary over #{primary_replication_host}" unless can_ping
+        end
+      rescue StandardError => e
+        validator_warnings << "Could not test private network connectivity from #{host}: #{e.message}"
+      end
+    end
+
+    def validate_system_requirements
+      puts '  Checking system requirements...'
+
+      all_hosts = [config.primary_host] + config.standby_hosts
+
+      # Capture errors and warnings before entering SSH block
+      validator_errors = errors
+      validator_warnings = warnings
+
+      all_hosts.each do |host|
+        ssh_executor.execute_on_host(host) do
+          # Check if running Debian/Ubuntu
+          unless test('[ -f /etc/debian_version ]')
+            validator_errors << "#{host} is not running Debian/Ubuntu"
+            next
+          end
+
+          # Check available disk space (require at least 10GB)
+          # Check /var or root filesystem (since /var/lib/postgresql may not exist yet)
+          disk_space_output = capture(:df, '-BG', '/var', '|', :tail, '-1', '|', :awk, "'{print $4}'")
+          disk_space = disk_space_output.gsub('G', '').to_i
+          validator_warnings << "#{host} has less than 10GB free disk space (#{disk_space}GB available)" if disk_space < 10
+
+          # Check if postgres user already exists
+          if test(:id, 'postgres')
+            validator_warnings << "#{host} already has a 'postgres' user - this is expected if PostgreSQL was previously installed"
+          end
+        end
+      rescue StandardError => e
+        validator_warnings << "Could not check system requirements on #{host}: #{e.message}"
+      end
+    end
+  end
+end

data/lib/active_postgres/version.rb

@@ -0,0 +1,3 @@
+module ActivePostgres
+  VERSION = '0.4.0'.freeze
+end