active_postgres 0.9.5 → 0.9.7

data/lib/active_postgres/components/repmgr.rb

@@ -1,3 +1,5 @@
+require 'shellwords'
+
 module ActivePostgres
   module Components
     class Repmgr < Base
@@ -22,31 +24,24 @@ module ActivePostgres
       def uninstall
         puts 'Uninstalling repmgr...'
 
-        config.all_hosts.each do |host|
-          is_primary = host == config.primary_host
-          node_id = if is_primary
-                      1
-                    else
-                      begin
-                        config.standby_hosts.index(host) + 2
-                      rescue StandardError
-                        999
-                      end
-                    end
+        config.all_hosts.each { |host| uninstall_from(host) }
+      end
 
-          ssh_executor.execute_on_host(host) do
-            cluster_output = begin
-              capture(:sudo, '-u', 'postgres', 'repmgr', 'cluster', 'show')
-            rescue StandardError
-              ''
-            end
+      def uninstall_from(host)
+        node_id = host == config.primary_host ? 1 : (config.standby_hosts.index(host) || 997) + 2
 
-            if cluster_output.include?("| #{node_id}") && cluster_output.include?('running')
-              info "✓ Skipping repmgr removal from #{host} - node #{node_id} is running"
-            else
-              info "Removing repmgr from #{host}"
-              execute :sudo, 'apt-get', 'remove', '-y', '-q', 'postgresql-*-repmgr', '||', 'true'
-            end
+        ssh_executor.execute_on_host(host) do
+          cluster_output = begin
+            capture(:sudo, '-u', 'postgres', 'repmgr', 'cluster', 'show')
+          rescue StandardError
+            ''
+          end
+
+          if cluster_output.include?("| #{node_id}") && cluster_output.include?('running')
+            info "✓ Skipping repmgr removal from #{host} - node #{node_id} is running"
+          else
+            info "Removing repmgr from #{host}"
+            execute :sudo, 'apt-get', 'remove', '-y', '-q', 'postgresql-*-repmgr', '||', 'true'
           end
         end
       end
@@ -62,7 +57,7 @@ module ActivePostgres
       end
 
       def setup_standby_only(standby_host)
-        puts "Setting up standby #{standby_host} (primary will not be touched)..."
+        puts "Setting up standby #{standby_host} (primary stays online)..."
 
         version = config.version
         install_apt_packages(standby_host, "postgresql-#{version}-repmgr")
@@ -103,7 +98,7 @@ module ActivePostgres
         puts '  Setting up primary with repmgr...'
 
         host = config.primary_host
-        repmgr_config = config.component_config(:repmgr)
+        repmgr_config = config.repmgr_config_for(host)
         version = config.version
         repmgr_password = normalize_repmgr_password(secrets.resolve('repmgr_password'))
         replication_password = normalize_replication_password(secrets.resolve('replication_password'))
@@ -114,6 +109,7 @@ module ActivePostgres
         if replication_user == repmgr_user && replication_password != repmgr_password
           raise Error, 'replication_user matches repmgr user but passwords differ. Use a distinct replication_user or the same password.'
         end
+
         effective_replication_password = replication_user == repmgr_user ? repmgr_password : replication_password
 
         # Variables used in ERB templates via binding
@@ -174,13 +170,13 @@ module ActivePostgres
           info 'Creating repmgr database and user...'
           repmgr_sql = repmgr_component.send(:build_repmgr_setup_sql, repmgr_user, repmgr_db, repmgr_password)
           executor.run_sql_on_backend(self, repmgr_sql, postgres_user: 'postgres', port: 5432, tuples_only: false,
-                                           capture: false)
+                                                        capture: false)
 
           if replication_user != repmgr_user
             info 'Ensuring replication user exists...'
             repl_sql = repmgr_component.send(:build_replication_user_sql, replication_user, effective_replication_password)
             executor.run_sql_on_backend(self, repl_sql, postgres_user: 'postgres', port: 5432, tuples_only: false,
-                                             capture: false)
+                                                        capture: false)
           end
 
           info 'Reloading PostgreSQL configuration to apply pg_hba.conf changes...'
@@ -202,14 +198,14 @@ module ActivePostgres
           cluster_show = nil
           5.times do |attempt|
             cluster_show = capture(:sudo, '-u', 'postgres', 'bash', '-lc',
-                                   "repmgr cluster show -f /etc/repmgr.conf 2>&1", raise_on_non_zero_exit: false).to_s
+                                   'repmgr cluster show -f /etc/repmgr.conf 2>&1', raise_on_non_zero_exit: false).to_s
 
             break if cluster_show.match?(/primary/i)
 
             sleep 2 if attempt < 4
           end
 
-          unless cluster_show && cluster_show.match?(/primary/i)
+          unless cluster_show&.match?(/primary/i)
             # Fallback: verify via repmgr metadata in the repmgr database
             db_check = executor.run_sql_on_backend(self,
                                                    'SELECT type FROM repmgr.nodes WHERE node_id = 1;',
@@ -235,7 +231,7 @@ module ActivePostgres
         puts "  Setting up standby: #{standby_host}..."
 
         host = standby_host
-        repmgr_config = config.component_config(:repmgr)
+        repmgr_config = config.repmgr_config_for(standby_host)
         primary_replication_host = config.primary_replication_host
         version = config.version
         secrets_obj = secrets
@@ -243,6 +239,11 @@ module ActivePostgres
         repmgr_db = config.repmgr_database
         postgres_user = config.postgres_user
         replication_user = config.replication_user
+        seed_method = config.standby_seed_method_for(standby_host)
+        restore_arguments = if seed_method == :pgbackrest
+                              standby_label = config.node_label_for(standby_host) || "standby-#{standby_host.split('.').first}"
+                              pgbackrest_restore_arguments(build_primary_conninfo(standby_label))
+                            end
 
         # Variables used in ERB templates via binding
         _ = host
@@ -255,13 +256,14 @@ module ActivePostgres
         if replication_user == repmgr_user && replication_password != repmgr_password
           raise Error, 'replication_user matches repmgr user but passwords differ. Use a distinct replication_user or the same password.'
         end
+
         effective_replication_password = replication_user == repmgr_user ? repmgr_password : replication_password
 
         ensure_primary_registered
         ensure_primary_replication_ready(repmgr_password, effective_replication_password)
 
         setup_pgpass_file(standby_host, repmgr_password, replication_password: effective_replication_password,
-                                                      primary_ip: primary_replication_host)
+                                                         primary_ip: primary_replication_host)
 
         if standby_already_configured?(standby_host)
           puts '  Standby already configured, updating configs...'
@@ -298,58 +300,61 @@ module ActivePostgres
             nil
           end
 
-          info 'Cloning from primary over private network...'
-          # Create a temporary repmgr config for cloning that points to the primary
-          # The regular config points to localhost which doesn't work during initial clone
-          temp_repmgr_conf = <<~CONF
-            node_id=#{node_id}
-            node_name='#{standby_host}'
-            conninfo='host=#{primary_replication_host} user=#{repmgr_user} dbname=#{repmgr_db} connect_timeout=10'
-            data_directory='/var/lib/postgresql/#{version}/main'
-          CONF
-
-          info 'Creating temporary repmgr config for cloning...'
-          upload! StringIO.new(temp_repmgr_conf), '/tmp/repmgr_clone.conf'
-          execute :sudo, 'mv', '/tmp/repmgr_clone.conf', '/etc/repmgr_clone.conf'
-          execute :sudo, 'chown', 'postgres:postgres', '/etc/repmgr_clone.conf'
-          execute :sudo, 'chmod', '600', '/etc/repmgr_clone.conf'
-
-          info 'Running: repmgr standby clone (this may take a few minutes to copy the database)...'
+          if seed_method == :repmgr
+            info 'Cloning from primary over private network...'
+            # Create a temporary repmgr config for cloning that points to the primary.
+            # The regular config points to localhost, which does not work during the initial clone.
+            temp_repmgr_conf = <<~CONF
+              node_id=#{node_id}
+              node_name='#{standby_host}'
+              conninfo='host=#{primary_replication_host} user=#{repmgr_user} dbname=#{repmgr_db} connect_timeout=10'
+              data_directory='/var/lib/postgresql/#{version}/main'
+            CONF
+
+            info 'Creating temporary repmgr config for cloning...'
+            upload! StringIO.new(temp_repmgr_conf), '/tmp/repmgr_clone.conf'
+            execute :sudo, 'mv', '/tmp/repmgr_clone.conf', '/etc/repmgr_clone.conf'
+            execute :sudo, 'chown', 'postgres:postgres', '/etc/repmgr_clone.conf'
+            execute :sudo, 'chmod', '600', '/etc/repmgr_clone.conf'
+            info 'Running: repmgr standby clone (this may take a few minutes to copy the database)...'
+          else
+            info 'Restoring the latest valid pgBackRest backup as a standby...'
+          end
 
           clone_success = false
           begin
-            execute :sudo, '-u', postgres_user, 'env',
-                    'HOME=/var/lib/postgresql',
-                    'repmgr', 'standby', 'clone',
-                    '-h', primary_replication_host,
-                    '-U', repmgr_user,
-                    '-d', repmgr_db,
-                    '-f', '/etc/repmgr_clone.conf',
-                    '--force',
-                    '--verbose'
-
-            info 'Clone command completed'
+            if seed_method == :pgbackrest
+              execute(*restore_arguments)
+              info 'pgBackRest standby restore completed'
+            else
+              execute :sudo, '-u', postgres_user, 'env',
+                      'HOME=/var/lib/postgresql',
+                      'repmgr', 'standby', 'clone',
+                      '-h', primary_replication_host,
+                      '-U', repmgr_user,
+                      '-d', repmgr_db,
+                      '-f', '/etc/repmgr_clone.conf',
+                      '--force',
+                      '--verbose'
+              info 'Clone command completed'
+            end
 
-            # Check if clone actually succeeded
             if test(:sudo, 'test', '-d', "/var/lib/postgresql/#{version}/main")
               clone_success = true
-              info 'Data directory successfully cloned!'
-
-              # Clean up temporary clone config
-              execute :sudo, 'rm', '-f', '/etc/repmgr_clone.conf'
+              info 'Standby data directory successfully seeded!'
+              execute :sudo, 'rm', '-f', '/etc/repmgr_clone.conf' if seed_method == :repmgr
             else
-              error "Data directory /var/lib/postgresql/#{version}/main was not created even though command succeeded!"
+              error "Data directory /var/lib/postgresql/#{version}/main was not created even though seeding succeeded!"
             end
           rescue SSHKit::Command::Failed => e
-            error "Clone command failed with exit code #{e.message}"
+            error "Standby seed command failed with exit code #{e.message}"
           rescue StandardError => e
-            error "Clone command raised exception: #{e.class}: #{e.message}"
+            error "Standby seed command raised exception: #{e.class}: #{e.message}"
           end
 
-          # If clone failed, raise error
           unless clone_success
-            error '✗ repmgr standby clone failed'
-            raise 'repmgr standby clone failed to create data directory'
+            error '✗ Standby data seeding failed'
+            raise 'standby data seeding failed to create data directory'
           end
         end
 
@@ -418,9 +423,10 @@ module ActivePostgres
 
       def setup_inter_node_ssh
         dns_config = dns_failover_config
-        key_path = dns_config && dns_config[:ssh_key_path] || '/var/lib/postgresql/.ssh/active_postgres_dns'
+        key_path = (dns_config && dns_config[:ssh_key_path]) || '/var/lib/postgresql/.ssh/active_postgres_dns'
         postgres_user = config.postgres_user
         all_hosts = config.all_hosts
+        replication_hosts = all_hosts.to_h { |host| [host, config.replication_host_for(host)] }
         pub_keys = {}
 
         all_hosts.each do |host|
@@ -430,18 +436,35 @@ module ActivePostgres
         all_hosts.each do |host|
           ssh_executor.execute_on_host(host) do
             other_keys = pub_keys.reject { |h, _| h == host }.values
-            other_keys.each do |key|
-              next if key.to_s.empty?
-
-              upload! StringIO.new("#{key}\n"), '/tmp/pg_peer_key.pub'
-              execute :sudo, '-u', postgres_user, 'bash', '-c',
-                      "grep -qxF -f /tmp/pg_peer_key.pub /var/lib/postgresql/.ssh/authorized_keys 2>/dev/null || cat /tmp/pg_peer_key.pub >> /var/lib/postgresql/.ssh/authorized_keys"
-              execute :rm, '-f', '/tmp/pg_peer_key.pub'
+            authorized_keys_path = '/var/lib/postgresql/.ssh/authorized_keys'
+            existing_keys = if test(:sudo, 'test', '-f', authorized_keys_path)
+                              capture(:sudo, 'cat', authorized_keys_path).lines.map(&:strip)
+                            else
+                              []
+                            end
+            authorized_keys = (existing_keys + other_keys.map(&:to_s).map(&:strip)).reject(&:empty?).uniq.join("\n")
+            unless authorized_keys.empty?
+              upload! StringIO.new("#{authorized_keys}\n"), '/tmp/pg_authorized_keys'
+              execute :sudo, 'mv', '/tmp/pg_authorized_keys', authorized_keys_path
+              execute :sudo, 'chown', "#{postgres_user}:#{postgres_user}", authorized_keys_path
+              execute :sudo, 'chmod', '600', authorized_keys_path
             end
 
-            peer_ips = all_hosts.reject { |h| h == host }.map { |h| config.replication_host_for(h) }
-            scan_cmd = "ssh-keyscan #{peer_ips.join(' ')} >> /var/lib/postgresql/.ssh/known_hosts 2>/dev/null || true"
-            execute :sudo, '-u', postgres_user, 'bash', '-c', scan_cmd
+            peer_ips = all_hosts.reject { |h| h == host }.map { |h| replication_hosts[h] }
+            known_hosts_path = '/var/lib/postgresql/.ssh/known_hosts'
+            existing_hosts = if test(:sudo, 'test', '-f', known_hosts_path)
+                               capture(:sudo, 'cat', known_hosts_path).lines.map(&:strip)
+                             else
+                               []
+                             end
+            scanned_hosts = capture(:sudo, '-u', postgres_user, 'ssh-keyscan', *peer_ips, raise_on_non_zero_exit: false)
+            known_hosts = (existing_hosts + scanned_hosts.lines.map(&:strip)).reject(&:empty?).uniq.join("\n")
+            unless known_hosts.empty?
+              upload! StringIO.new("#{known_hosts}\n"), '/tmp/pg_known_hosts'
+              execute :sudo, 'mv', '/tmp/pg_known_hosts', known_hosts_path
+              execute :sudo, 'chown', "#{postgres_user}:#{postgres_user}", known_hosts_path
+              execute :sudo, 'chmod', '600', known_hosts_path
+            end
           end
         end
       end
@@ -518,10 +541,21 @@ module ActivePostgres
           public_key = capture(:sudo, '-u', postgres_user, 'cat', "#{key_path}.pub").strip
 
           unless dns_servers.empty?
-            execute :sudo, '-u', postgres_user, 'touch', '/var/lib/postgresql/.ssh/known_hosts'
-            scan_cmd = "ssh-keyscan -H #{dns_servers.join(' ')} >> /var/lib/postgresql/.ssh/known_hosts 2>/dev/null || true"
-            execute :sudo, '-u', postgres_user, 'bash', '-c', scan_cmd
-            execute :sudo, '-u', postgres_user, 'chmod', '600', '/var/lib/postgresql/.ssh/known_hosts'
+            known_hosts_path = '/var/lib/postgresql/.ssh/known_hosts'
+            existing_hosts = if test(:sudo, 'test', '-f', known_hosts_path)
+                               capture(:sudo, 'cat', known_hosts_path).lines.map(&:strip)
+                             else
+                               []
+                             end
+            scanned_hosts = capture(:sudo, '-u', postgres_user, 'ssh-keyscan', '-H', *dns_servers,
+                                    raise_on_non_zero_exit: false)
+            known_hosts = (existing_hosts + scanned_hosts.lines.map(&:strip)).reject(&:empty?).uniq.join("\n")
+            unless known_hosts.empty?
+              upload! StringIO.new("#{known_hosts}\n"), '/tmp/pg_known_hosts'
+              execute :sudo, 'mv', '/tmp/pg_known_hosts', known_hosts_path
+              execute :sudo, 'chown', "#{postgres_user}:#{postgres_user}", known_hosts_path
+              execute :sudo, 'chmod', '600', known_hosts_path
+            end
           end
         end
 
@@ -542,8 +576,8 @@ module ActivePostgres
 
             upload! StringIO.new("#{key}\n"), '/tmp/active_postgres_dns_key.pub'
             execute :bash, '-c',
-                    "grep -qxF -f /tmp/active_postgres_dns_key.pub ~/.ssh/authorized_keys || " \
-                    "cat /tmp/active_postgres_dns_key.pub >> ~/.ssh/authorized_keys"
+                    'grep -qxF -f /tmp/active_postgres_dns_key.pub ~/.ssh/authorized_keys || ' \
+                    'cat /tmp/active_postgres_dns_key.pub >> ~/.ssh/authorized_keys'
             execute :rm, '-f', '/tmp/active_postgres_dns_key.pub'
           end
         end
@@ -560,11 +594,11 @@ module ActivePostgres
 
         domains = normalize_dns_domains(dns_config)
         primary_records = normalize_dns_records(dns_config[:primary_records] || dns_config[:primary_record],
-                                               default_prefix: 'db-primary',
-                                               domains: domains)
+                                                default_prefix: 'db-primary',
+                                                domains: domains)
         replica_records = normalize_dns_records(dns_config[:replica_records] || dns_config[:replica_record],
-                                               default_prefix: 'db-replica',
-                                               domains: domains)
+                                                default_prefix: 'db-replica',
+                                                domains: domains)
 
         _ = primary_records
         _ = replica_records
@@ -732,10 +766,10 @@ module ActivePostgres
             # Check replication status
             begin
               rep_status = executor.run_sql_on_backend(self,
-                                                      'SELECT pg_is_in_recovery();',
-                                                      postgres_user: postgres_user,
-                                                      tuples_only: true,
-                                                      capture: true).to_s
+                                                       'SELECT pg_is_in_recovery();',
+                                                       postgres_user: postgres_user,
+                                                       tuples_only: true,
+                                                       capture: true).to_s
               if rep_status.include?('t')
                 info '✓ Standby is in recovery mode (receiving replication)'
               else
@@ -750,10 +784,10 @@ module ActivePostgres
             # Check lag
             begin
               lag_result = executor.run_sql_on_backend(self,
-                                                      'SELECT EXTRACT(EPOCH FROM (now() - pg_last_xact_replay_timestamp()))::int AS lag;',
-                                                      postgres_user: postgres_user,
-                                                      tuples_only: true,
-                                                      capture: true).to_s.strip
+                                                       'SELECT EXTRACT(EPOCH FROM (now() - pg_last_xact_replay_timestamp()))::int AS lag;',
+                                                       postgres_user: postgres_user,
+                                                       tuples_only: true,
+                                                       capture: true).to_s.strip
               info "Replication lag: #{lag_result}"
             rescue StandardError => e
               error "Failed to get replication lag: #{e.message}"
@@ -785,7 +819,7 @@ module ActivePostgres
         # Create .pgpass file for postgres user to avoid password exposure in logs
         # Format: hostname:port:database:username:password
         pgpass_content = build_pgpass_content(host, repmgr_password, replication_password: replication_password,
-                                                                   primary_ip: primary_ip)
+                                                                     primary_ip: primary_ip)
 
         ssh_executor.execute_on_host(host) do
           # Create .pgpass in postgres user's home directory
@@ -854,8 +888,13 @@ module ActivePostgres
       def escape_pgpass_value(value)
         return '' if value.nil?
 
-        # Must escape backslashes first, then colons (order matters!)
-        value.to_s.gsub('\\', '\\\\\\\\').gsub(':', '\\:')
+        value.to_s.each_char.map do |char|
+          case char
+          when '\\' then '\\\\'
+          when ':' then '\\:'
+          else char
+          end
+        end.join
       end
 
       def build_primary_conninfo(standby_label)
@@ -871,6 +910,21 @@ module ActivePostgres
         "host=#{primary_host} user=#{user} dbname=#{dbname} application_name=#{standby_label}"
       end
 
+      def pgbackrest_restore_arguments(primary_conninfo)
+        command = Shellwords.join(
+          [
+            'pgbackrest', '--stanza=main', '--type=standby',
+            "--recovery-option=primary_conninfo=#{primary_conninfo}",
+            'restore'
+          ]
+        )
+
+        [
+          :sudo, '-u', config.postgres_user,
+          :bash, '-lc', Shellwords.escape(command)
+        ]
+      end
+
       def normalize_repmgr_password(raw_password)
         password = raw_password.to_s.rstrip
 
@@ -935,12 +989,12 @@ module ActivePostgres
         ssh_executor.execute_on_host(host) do
           repmgr_sql = repmgr_component.send(:build_repmgr_setup_sql, repmgr_user, repmgr_db, repmgr_password)
           executor.run_sql_on_backend(self, repmgr_sql, postgres_user: 'postgres', port: 5432, tuples_only: false,
-                                           capture: false)
+                                                        capture: false)
 
           if replication_user != repmgr_user
             repl_sql = repmgr_component.send(:build_replication_user_sql, replication_user, effective_replication_password)
             executor.run_sql_on_backend(self, repl_sql, postgres_user: 'postgres', port: 5432, tuples_only: false,
-                                             capture: false)
+                                                        capture: false)
           end
 
           info '✓ Primary replication user is ready'
@@ -1065,7 +1119,13 @@ module ActivePostgres
       end
 
       def enable_repmgrd_if_configured(host, repmgr_config)
-        return if repmgr_config[:auto_failover] == false
+        if repmgr_config[:auto_failover] == false
+          ssh_executor.execute_on_host(host) do
+            execute :sudo, 'systemctl', 'stop', 'repmgrd', raise_on_non_zero_exit: false
+            execute :sudo, 'systemctl', 'disable', 'repmgrd', raise_on_non_zero_exit: false
+          end
+          return
+        end
 
         default_config = <<~CONF
           REPMGRD_ENABLED=yes

data/lib/active_postgres/configuration/pgbackrest_policy.rb

@@ -0,0 +1,19 @@
+module ActivePostgres
+  class Configuration
+    module PgBackRestPolicy
+      private
+
+      def validate_pgbackrest!
+        return unless component_enabled?(:pgbackrest)
+
+        pg_config = component_config(:pgbackrest)
+        retention_full = pg_config[:retention_full]
+        retention_archive = pg_config[:retention_archive]
+        if retention_full && retention_archive && retention_archive.to_i < retention_full.to_i
+          raise Error, 'pgbackrest.retention_archive must be >= retention_full for PITR safety'
+        end
+        raise Error, 'pgbackrest.repo_block requires repo_bundle' if pg_config[:repo_block] && !pg_config[:repo_bundle]
+      end
+    end
+  end
+end

data/lib/active_postgres/configuration/standby_policy.rb

@@ -0,0 +1,24 @@
+module ActivePostgres
+  class Configuration
+    module StandbyPolicy
+      def standby_seed_method_for(host)
+        (standby_config_for(host)&.fetch('seed_method', nil) || 'repmgr').to_s.to_sym
+      end
+
+      private
+
+      def validate_standby_seed_methods!
+        @standbys.each do |standby|
+          host = standby.fetch('host')
+          seed_method = standby_seed_method_for(host)
+          unless %i[repmgr pgbackrest].include?(seed_method)
+            raise Error, "Invalid seed_method '#{seed_method}' for standby #{host}. Use 'repmgr' or 'pgbackrest'."
+          end
+          if seed_method == :pgbackrest && !component_enabled?(:pgbackrest)
+            raise Error, "Standby #{host} requires the pgbackrest component for seed_method 'pgbackrest'"
+          end
+        end
+      end
+    end
+  end
+end