active_postgres 0.9.2 → 0.9.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: e51d5f1214cedf497a9b060e5c667c43711f9ff5e36024bfd8dc926b50fe272f
-  data.tar.gz: 618fa70d39a43e1c74ba2ed449b75138670bca92cb6bf418606f772de34c6a7d
+  metadata.gz: b0966aed96fa6f906e7b048ef7804a0b9994c54f841f8dc9f00425f57f565a71
+  data.tar.gz: 86582950fc4afa733cba7052682dbaf0f3e76275c9397559f2d3398c8f92da86
 SHA512:
-  metadata.gz: 254b6fb4342d4e510076d05ebfbeaa0e7990b463df13ad6ddb7d6eb5a70562c23cc4f502ce6c435c6372fc3dd5f695285ee3fdfed522cc188f1b1cc21c6cc90d
-  data.tar.gz: d9147a4679529344ccb0ffd3d5c2ffacc3072464b9cf4e1c6e11150a66c142ef0d0b45f9b9cd9d34eceba6f4d0e50baa7c2f227a5b2b1719502972e2b4b39ee4
+  metadata.gz: 18644b19453727a1b14918f9bc7d055fdf2c8cbaab962d94acdb058be4fa6b969e72e569ea5e1a2f0848cc45433e35c47dfa69af699370cb267b64c0a61114d5
+  data.tar.gz: '08b112f2121747a9b970809bf0c662dfbed99166edf6e3b18e88daab4fff0a65fcae13821c37fa69af218497b9ac05978d2bf6cd157c197d0f719f968088f407'

data/README.md

@@ -187,12 +187,10 @@ rake postgres:backup:restore_at["2026-01-29 01:15:00",promote]
 ```
 
 If you want the old primary to rejoin without a full re-clone after failover,
-enable pg_rewind support in repmgr:
+use `repmgr node rejoin --force-rewind` once the node is ready to reattach:
 
-```yaml
-components:
-  repmgr:
-    use_rewind: true
+```bash
+repmgr node rejoin -f /etc/repmgr.conf --force-rewind
 ```
 
 ### Stable app endpoint with PgBouncer

data/lib/active_postgres/components/base.rb

@@ -43,6 +43,14 @@ module ActivePostgres
         content = render_template(template_name, binding_context)
         ssh_executor.upload_file(host, content, remote_path, mode: mode, owner: owner)
       end
+
+      def install_apt_packages(host, *packages)
+        ssh_executor.execute_on_host(host) do
+          execute :sudo, 'DEBIAN_FRONTEND=noninteractive', 'apt-get',
+                  '-o', 'DPkg::Lock::Timeout=300',
+                  'install', '-y', '-qq', *packages
+        end
+      end
     end
   end
 end

data/lib/active_postgres/components/extensions.rb

@@ -65,10 +65,9 @@ module ActivePostgres
         return if packages_to_install.empty?
 
         ssh_executor.execute_on_host(host) do
-          execute :sudo, 'apt-get', 'update', '-qq'
-          execute :sudo, 'DEBIAN_FRONTEND=noninteractive', 'apt-get', 'install', '-y', '-qq',
-                  *packages_to_install
+          execute :sudo, 'apt-get', '-o', 'DPkg::Lock::Timeout=300', 'update', '-qq'
         end
+        install_apt_packages(host, *packages_to_install)
       end
 
       def install_on_primary(extensions)

data/lib/active_postgres/components/monitoring.rb

@@ -60,10 +60,7 @@ module ActivePostgres
         exporter_port = monitoring_config[:exporter_port] || 9187
 
         # Download and install postgres_exporter
-        ssh_executor.execute_on_host(host) do
-          # Install via package manager or download binary
-          execute :sudo, 'apt-get', 'install', '-y', '-qq', 'prometheus-postgres-exporter'
-        end
+        install_apt_packages(host, 'prometheus-postgres-exporter')
 
         configure_exporter_service(host, monitoring_config, exporter_port)
 
@@ -157,9 +154,7 @@ module ActivePostgres
         listen_address = monitoring_config[:node_exporter_listen_address].to_s.strip
         puts "  Installing node_exporter on #{host}..."
 
-        ssh_executor.execute_on_host(host) do
-          execute :sudo, 'apt-get', 'install', '-y', '-qq', 'prometheus-node-exporter'
-        end
+        install_apt_packages(host, 'prometheus-node-exporter')
 
         configure_node_exporter_service(host, port, listen_address)
 
@@ -213,12 +208,15 @@ module ActivePostgres
         puts "Installing Grafana on #{host}..."
 
         ssh_executor.execute_on_host(host) do
-          execute :sudo, 'apt-get', 'install', '-y', '-qq', 'apt-transport-https', 'software-properties-common', 'wget'
+          execute :sudo, 'DEBIAN_FRONTEND=noninteractive', 'apt-get',
+                  '-o', 'DPkg::Lock::Timeout=300', 'install', '-y', '-qq',
+                  'apt-transport-https', 'software-properties-common', 'wget'
           execute :sudo, 'wget', '-q', '-O', '/usr/share/keyrings/grafana.gpg', 'https://apt.grafana.com/gpg.key'
           execute :sudo, 'sh', '-c',
                   'echo "deb [signed-by=/usr/share/keyrings/grafana.gpg] https://apt.grafana.com stable main" > /etc/apt/sources.list.d/grafana.list'
-          execute :sudo, 'apt-get', 'update', '-qq'
-          execute :sudo, 'apt-get', 'install', '-y', '-qq', 'grafana'
+          execute :sudo, 'apt-get', '-o', 'DPkg::Lock::Timeout=300', 'update', '-qq'
+          execute :sudo, 'DEBIAN_FRONTEND=noninteractive', 'apt-get',
+                  '-o', 'DPkg::Lock::Timeout=300', 'install', '-y', '-qq', 'grafana'
           execute :sudo, 'systemctl', 'enable', '--now', 'grafana-server'
           execute :sudo, 'grafana-cli', 'admin', 'reset-admin-password', admin_password
         end

data/lib/active_postgres/components/pgbackrest.rb

@@ -1,6 +1,12 @@
+require 'shellwords'
+
 module ActivePostgres
   module Components
     class PgBackRest < Base
+      LOG_ARCHIVE_CRON_FILE = '/etc/cron.d/postgres-log-archive'
+      LOG_ARCHIVE_ENV_FILE = '/etc/active-postgres-log-archive.env'
+      LOG_ARCHIVE_SCRIPT = '/usr/local/bin/active-postgres-archive-logs'
+
       def install
         puts 'Installing pgBackRest for backups...'
 
@@ -21,6 +27,9 @@ module ActivePostgres
             execute :sudo, 'apt-get', 'remove', '-y', 'pgbackrest'
             execute :sudo, 'rm', '-f', '/etc/cron.d/pgbackrest-backup'
             execute :sudo, 'rm', '-f', '/etc/cron.d/pgbackrest-backup-incremental'
+            execute :sudo, 'rm', '-f', LOG_ARCHIVE_CRON_FILE
+            execute :sudo, 'rm', '-f', LOG_ARCHIVE_ENV_FILE
+            execute :sudo, 'rm', '-f', LOG_ARCHIVE_SCRIPT
           end
         end
       end
@@ -105,10 +114,7 @@ module ActivePostgres
         _ = pgbackrest_config # Used in ERB template
         _ = secrets_obj # Used in ERB template
 
-        # Install package
-        ssh_executor.execute_on_host(host) do
-          execute :sudo, 'apt-get', 'install', '-y', '-qq', 'pgbackrest'
-        end
+        install_apt_packages(host, 'pgbackrest')
 
         # Upload configuration
         upload_template(host, 'pgbackrest.conf.erb', '/etc/pgbackrest.conf', binding,
@@ -134,13 +140,22 @@ module ActivePostgres
           end
         end
 
+        if log_archive_enabled?(pgbackrest_config)
+          setup_log_archive(host, pgbackrest_config)
+        else
+          remove_log_archive(host)
+        end
+
         # Set up scheduled backups on primary only
         if create_stanza
           setup_backup_schedules(host, pgbackrest_config)
+        else
+          remove_backup_schedule(host)
         end
       end
 
       def setup_backup_schedules(host, pgbackrest_config)
+        remove_backup_schedule(host)
         schedules = backup_schedules(pgbackrest_config)
         schedules.each do |entry|
           setup_backup_schedule(host, entry[:schedule], entry[:type], entry[:file])
@@ -186,6 +201,75 @@ module ActivePostgres
           execute :sudo, 'rm', '-f', '/etc/cron.d/pgbackrest-backup-incremental'
         end
       end
+
+      def setup_log_archive(host, pgbackrest_config)
+        log_archive_config = pgbackrest_config[:log_archive] || {}
+        unless pgbackrest_config[:repo_type] == 's3'
+          raise Error, 'pgbackrest.log_archive requires pgbackrest.repo_type: s3'
+        end
+
+        postgres_user = config.postgres_user
+        env_content = log_archive_env(pgbackrest_config, log_archive_config, host)
+        cron_content = log_archive_cron(log_archive_config)
+
+        install_apt_packages(host, 'awscli')
+
+        ssh_executor.upload_file(host, env_content, LOG_ARCHIVE_ENV_FILE, mode: '640', owner: "root:#{postgres_user}")
+
+        upload_template(host, 'postgres_log_archive.sh.erb', LOG_ARCHIVE_SCRIPT, binding,
+                        mode: '750', owner: "root:#{postgres_user}")
+        ssh_executor.upload_file(host, cron_content, LOG_ARCHIVE_CRON_FILE, mode: '644', owner: 'root:root')
+      end
+
+      def remove_log_archive(host)
+        ssh_executor.execute_on_host(host) do
+          execute :sudo, 'rm', '-f', LOG_ARCHIVE_CRON_FILE
+          execute :sudo, 'rm', '-f', LOG_ARCHIVE_ENV_FILE
+          execute :sudo, 'rm', '-f', LOG_ARCHIVE_SCRIPT
+        end
+      end
+
+      def log_archive_enabled?(pgbackrest_config)
+        (pgbackrest_config[:log_archive] || {})[:enabled] == true
+      end
+
+      def log_archive_cron(log_archive_config)
+        schedule = log_archive_config[:schedule] || '17 3 * * *'
+        <<~CRON
+          # PostgreSQL text log archive (managed by active_postgres)
+          SHELL=/bin/bash
+          PATH=/usr/local/sbin:/usr/local/bin:/sbin:/bin:/usr/sbin:/usr/bin
+          #{schedule} root #{LOG_ARCHIVE_SCRIPT}
+        CRON
+      end
+
+      def log_archive_env(pgbackrest_config, log_archive_config, host)
+        access_key = secrets.resolve('s3_access_key')
+        secret_key = secrets.resolve('s3_secret_key')
+        raise Error, 's3_access_key secret is required for pgbackrest.log_archive' if access_key.to_s.empty?
+        raise Error, 's3_secret_key secret is required for pgbackrest.log_archive' if secret_key.to_s.empty?
+
+        env = {
+          'AWS_ACCESS_KEY_ID' => access_key,
+          'AWS_SECRET_ACCESS_KEY' => secret_key,
+          'AWS_DEFAULT_REGION' => pgbackrest_config[:s3_region] || 'us-east-1',
+          'AWS_BUCKET' => pgbackrest_config[:s3_bucket],
+          'AWS_ENDPOINT_URL' => s3_endpoint_url(pgbackrest_config),
+          'POSTGRES_LOG_ARCHIVE_LOG_DIR' => log_archive_config[:log_directory] || '/var/log/postgresql',
+          'POSTGRES_LOG_ARCHIVE_PREFIX' => log_archive_config[:prefix] || 'postgres-logs',
+          'POSTGRES_LOG_ARCHIVE_NODE' => config.node_label_for(host) || host
+        }.compact
+
+        env.map { |key, value| "#{key}=#{Shellwords.escape(value.to_s)}" }.join("\n") + "\n"
+      end
+
+      def s3_endpoint_url(pgbackrest_config)
+        endpoint = pgbackrest_config[:s3_endpoint]
+        return nil if endpoint.to_s.empty?
+        return endpoint if endpoint.match?(/\Ahttps?:\/\//)
+
+        "https://#{endpoint}"
+      end
     end
   end
 end

data/lib/active_postgres/components/pgbouncer.rb

@@ -60,15 +60,7 @@ module ActivePostgres
 
         user_config = config.component_config(:pgbouncer)
 
-        # Determine follow_primary behavior:
-        # - Primary: use global follow_primary setting
-        # - Standby: check per-standby pgbouncer_follow_primary, default false (use localhost for read replicas)
-        if is_standby
-          standby_config = config.standby_config_for(host) || {}
-          follow_primary = standby_config['pgbouncer_follow_primary'] == true
-        else
-          follow_primary = user_config[:follow_primary] == true
-        end
+        follow_primary = follow_primary_for?(host, is_standby: is_standby, user_config: user_config)
 
         if follow_primary && !config.component_enabled?(:repmgr)
           raise Error, 'PgBouncer follow_primary requires repmgr to be enabled'
@@ -89,9 +81,7 @@ module ActivePostgres
 
         puts "  Calculated pool settings for max_connections=#{max_connections}"
 
-        ssh_executor.execute_on_host(host) do
-          execute :sudo, 'apt-get', 'install', '-y', '-qq', 'pgbouncer'
-        end
+        install_apt_packages(host, 'pgbouncer')
 
         upload_template(host, 'pgbouncer.ini.erb', '/etc/pgbouncer/pgbouncer.ini', binding, mode: '644')
 
@@ -99,14 +89,34 @@ module ActivePostgres
 
         create_userlist(host)
 
+        ensure_firewall_port_open(host, pgbouncer_config[:listen_port] || 6432)
+
         ssh_executor.execute_on_host(host) do
           execute :sudo, 'systemctl', 'enable', 'pgbouncer'
           execute :sudo, 'systemctl', 'restart', 'pgbouncer'
+          unless follow_primary
+            execute :sudo, 'systemctl', 'disable', '--now', 'pgbouncer-follow-primary.timer', '||', 'true'
+            execute :sudo, 'rm', '-f', '/usr/local/bin/pgbouncer-follow-primary',
+                    '/etc/systemd/system/pgbouncer-follow-primary.service',
+                    '/etc/systemd/system/pgbouncer-follow-primary.timer'
+            execute :sudo, 'systemctl', 'daemon-reload'
+          end
         end
 
         install_follow_primary(host, pgbouncer_config) if follow_primary
       end
 
+      def follow_primary_for?(host, is_standby:, user_config:)
+        if is_standby
+          standby_config = config.standby_config_for(host) || {}
+          standby_override = standby_config['pgbouncer_follow_primary']
+          standby_override = standby_config[:pgbouncer_follow_primary] if standby_override.nil?
+          return standby_override == true unless standby_override.nil?
+        end
+
+        user_config[:follow_primary] == true || user_config['follow_primary'] == true
+      end
+
       def setup_ssl_certs(host)
         puts '  Setting up SSL certificates for PgBouncer...'
         version = config.version
@@ -194,15 +204,28 @@ module ActivePostgres
         end
       end
 
+      def ensure_firewall_port_open(host, port)
+        ssh_executor.execute_on_host(host) do
+          has_reject = test(:sudo, 'iptables', '-C', 'INPUT', '-j', 'REJECT', '2>/dev/null')
+          if has_reject
+            execute :sudo, 'iptables', '-I', 'INPUT', '-p', 'tcp', '--dport', port.to_s, '-j', 'ACCEPT'
+            execute :sudo, 'sh', '-c', "'iptables-save > /etc/iptables/rules.v4 2>/dev/null || true'"
+            puts "  ✓ Opened port #{port} in iptables"
+          end
+        end
+      end
+
       def install_follow_primary(host, pgbouncer_config)
         interval = pgbouncer_config[:follow_primary_interval] || 5
         interval = interval.to_i
         interval = 5 if interval <= 0
 
         repmgr_conf = '/etc/repmgr.conf'
+        repmgr_database = config.repmgr_database
         postgres_user = config.postgres_user
         _ = interval
         _ = repmgr_conf
+        _ = repmgr_database
         _ = postgres_user
 
         upload_template(host, 'pgbouncer_follow_primary.sh.erb', '/usr/local/bin/pgbouncer-follow-primary', binding,

data/lib/active_postgres/components/repmgr.rb

@@ -11,6 +11,7 @@ module ActivePostgres
           setup_standby(host)
         end
 
+        setup_inter_node_ssh if config.standby_hosts.any?
         setup_dns_failover if dns_failover_enabled?
 
         # Verify the entire cluster is healthy after setup
@@ -64,10 +65,7 @@ module ActivePostgres
         puts "Setting up standby #{standby_host} (primary will not be touched)..."
 
         version = config.version
-        ssh_executor.execute_on_host(standby_host) do
-          execute :sudo, 'DEBIAN_FRONTEND=noninteractive', 'apt-get', 'install', '-y', '-qq',
-                  "postgresql-#{version}-repmgr"
-        end
+        install_apt_packages(standby_host, "postgresql-#{version}-repmgr")
 
         setup_standby(standby_host)
 
@@ -81,10 +79,7 @@ module ActivePostgres
 
         version = config.version
         config.all_hosts.each do |host|
-          ssh_executor.execute_on_host(host) do
-            execute :sudo, 'DEBIAN_FRONTEND=noninteractive', 'apt-get', 'install', '-y', '-qq',
-                    "postgresql-#{version}-repmgr"
-          end
+          install_apt_packages(host, "postgresql-#{version}-repmgr")
           install_postgres_sudoers(host)
         end
       end
@@ -275,6 +270,7 @@ module ActivePostgres
           update_postgres_configs_on_standby(standby_host, version)
           ensure_ssl_certs(standby_host, version) if config.component_enabled?(:ssl)
           register_standby_with_primary(standby_host)
+          setup_inter_node_ssh
           enable_repmgrd_if_configured(standby_host, repmgr_config)
           return
         end
@@ -428,9 +424,7 @@ module ActivePostgres
         pub_keys = {}
 
         all_hosts.each do |host|
-          ssh_executor.execute_on_host(host) do
-            pub_keys[host] = capture(:sudo, '-u', postgres_user, 'cat', "#{key_path}.pub").strip
-          end
+          pub_keys[host] = ensure_dns_ssh_key(host, key_path, [])
         end
 
         all_hosts.each do |host|
@@ -1073,13 +1067,23 @@ module ActivePostgres
       def enable_repmgrd_if_configured(host, repmgr_config)
         return if repmgr_config[:auto_failover] == false
 
+        default_config = <<~CONF
+          REPMGRD_ENABLED=yes
+          REPMGRD_CONF="/etc/repmgr.conf"
+          REPMGRD_OPTS=""
+          REPMGRD_USER=postgres
+          REPMGRD_BIN=/usr/bin/repmgrd
+          REPMGRD_PIDFILE=/var/run/repmgrd.pid
+        CONF
+
         ssh_executor.execute_on_host(host) do
-          begin
-            execute :sudo, 'systemctl', 'enable', 'repmgrd'
-            execute :sudo, 'systemctl', 'restart', 'repmgrd'
-          rescue StandardError
-            nil
-          end
+          upload! StringIO.new(default_config), '/tmp/repmgrd-default'
+          execute :sudo, 'mv', '/tmp/repmgrd-default', '/etc/default/repmgrd'
+          execute :sudo, 'chown', 'root:root', '/etc/default/repmgrd'
+          execute :sudo, 'chmod', '644', '/etc/default/repmgrd'
+          execute :sudo, 'systemctl', 'enable', 'repmgrd'
+          execute :sudo, 'systemctl', 'restart', 'repmgrd'
+          execute :pgrep, '-x', 'repmgrd'
         end
       end
 

data/lib/active_postgres/connection_pooler.rb

@@ -240,8 +240,8 @@ module ActivePostgres
 
       @ssh_executor.execute_on_host(host) do
         # Install PgBouncer package
-        execute :sudo, 'DEBIAN_FRONTEND=noninteractive', 'apt-get', 'install',
-                '-y', '-qq', 'pgbouncer'
+        execute :sudo, 'DEBIAN_FRONTEND=noninteractive', 'apt-get',
+                '-o', 'DPkg::Lock::Timeout=300', 'install', '-y', '-qq', 'pgbouncer'
 
         # Create required directories
         execute :sudo, 'mkdir', '-p', '/var/log/pgbouncer'

data/lib/active_postgres/generators/active_postgres/install_generator.rb

@@ -79,7 +79,7 @@ module ActivePostgres
             default: &default
               adapter: postgresql
               encoding: unicode
-              pool: <%= ENV.fetch("RAILS_MAX_THREADS") { 5 } %>
+              pool: <%= ENV.fetch("DB_POOL") { ENV.fetch("RAILS_MAX_THREADS") { 5 } } %>
 
             development:
               <<: *default

data/lib/active_postgres/generators/active_postgres/templates/postgres.yml.erb

@@ -50,6 +50,8 @@ shared: &shared
       enabled: false
       # Optional: Override default pgbouncer user
       # user: pgbouncer
+      # Keep PgBouncer on every node pointed at the current primary.
+      # Set standby.pgbouncer_follow_primary: false on a specific standby to keep it local.
       # follow_primary: true
       # follow_primary_interval: 5
     

data/lib/active_postgres/rails/database_config.rb

@@ -82,7 +82,7 @@ module ActivePostgres
           {
             'adapter' => 'postgresql',
             'encoding' => 'unicode',
-            'pool' => env_value('RAILS_MAX_THREADS', fallback: 5),
+            'pool' => pool_env_value,
             'username' => env_value('POSTGRES_APP_USER', fallback: app_name),
             'password' => env_value('POSTGRES_APP_PASSWORD'),
             'variables' => {
@@ -138,6 +138,10 @@ module ActivePostgres
           end
         end
 
+        def pool_env_value
+          "<%= ENV.fetch('DB_POOL') { ENV.fetch('RAILS_MAX_THREADS') { 5 } } %>"
+        end
+
         def normalize_app_name(custom_name)
           chosen = custom_name || ENV['BORING_APP_NAME'] || default_app_name
           chosen.to_s.tr('- ', '_')

data/lib/active_postgres/ssh_executor.rb

@@ -50,9 +50,10 @@ module ActivePostgres
                   '/etc/apt/sources.list.d/pgdg.list'
         end
 
-        execute :sudo, 'apt-get', 'update', '-qq'
-        execute :sudo, 'DEBIAN_FRONTEND=noninteractive', 'apt-get', 'install', '-y', '-qq', 'gnupg', 'wget',
-                'lsb-release', 'locales'
+        execute :sudo, 'apt-get', '-o', 'DPkg::Lock::Timeout=300', 'update', '-qq'
+        execute :sudo, 'DEBIAN_FRONTEND=noninteractive', 'apt-get',
+                '-o', 'DPkg::Lock::Timeout=300', 'install', '-y', '-qq',
+                'gnupg', 'wget', 'lsb-release', 'locales'
 
         info 'Generating locales...'
         execute :sudo, 'locale-gen', 'en_US.UTF-8'
@@ -73,8 +74,9 @@ module ActivePostgres
             execute :sudo, 'systemctl', 'stop', 'postgresql' if test('systemctl is-active postgresql')
 
             # Remove all PostgreSQL packages
-            execute :sudo, 'DEBIAN_FRONTEND=noninteractive', 'apt-get', 'remove', '--purge', '-y', '-qq', 'postgresql*'
-            execute :sudo, 'apt-get', 'autoremove', '-y', '-qq'
+            execute :sudo, 'DEBIAN_FRONTEND=noninteractive', 'apt-get',
+                    '-o', 'DPkg::Lock::Timeout=300', 'remove', '--purge', '-y', '-qq', 'postgresql*'
+            execute :sudo, 'apt-get', '-o', 'DPkg::Lock::Timeout=300', 'autoremove', '-y', '-qq'
 
             # Clean up OLD version directories only (preserve target version SSL certs, etc.)
             other_versions.each do |old_version|
@@ -103,9 +105,10 @@ module ActivePostgres
                     "/etc/apt/sources.list.d/pgdg.list'"
         execute :sudo, 'sh', '-c', pgdg_repo
 
-        execute :sudo, 'apt-get', 'update', '-qq'
-        execute :sudo, 'DEBIAN_FRONTEND=noninteractive', 'apt-get', 'install', '-y', '-qq', "postgresql-#{version}",
-                "postgresql-client-#{version}"
+        execute :sudo, 'apt-get', '-o', 'DPkg::Lock::Timeout=300', 'update', '-qq'
+        execute :sudo, 'DEBIAN_FRONTEND=noninteractive', 'apt-get',
+                '-o', 'DPkg::Lock::Timeout=300', 'install', '-y', '-qq',
+                "postgresql-#{version}", "postgresql-client-#{version}"
 
         execute :sudo, 'systemctl', 'enable', 'postgresql'
         execute :sudo, 'systemctl', 'start', 'postgresql' unless test('systemctl is-active postgresql')

data/lib/active_postgres/version.rb

@@ -1,3 +1,3 @@
 module ActivePostgres
-  VERSION = '0.9.2'.freeze
+  VERSION = '0.9.4'.freeze
 end

data/lib/tasks/postgres.rake

@@ -97,13 +97,14 @@ namespace :postgres do
 
         # Remove packages
         begin
-          execute :sudo, 'DEBIAN_FRONTEND=noninteractive', 'apt-get', 'remove', '--purge', '-y',
+          execute :sudo, 'DEBIAN_FRONTEND=noninteractive', 'apt-get',
+                  '-o', 'DPkg::Lock::Timeout=300', 'remove', '--purge', '-y',
                   'postgresql*', 'pgbouncer', 'repmgr', 'prometheus-postgres-exporter'
         rescue StandardError
           nil
         end
         begin
-          execute :sudo, 'apt-get', 'autoremove', '-y'
+          execute :sudo, 'apt-get', '-o', 'DPkg::Lock::Timeout=300', 'autoremove', '-y'
         rescue StandardError
           nil
         end

data/lib/tasks/rolling_update.rake

@@ -56,8 +56,10 @@ namespace :postgres do
       puts "\nUpdating #{standby}..."
       ssh_executor.execute_on_host(standby) do
         execute :sudo, 'systemctl', 'stop', "postgresql@#{current_version}-main"
-        execute :sudo, 'apt-get', 'update'
-        execute :sudo, 'apt-get', 'install', '-y', "postgresql-#{new_version}", "postgresql-contrib-#{new_version}"
+        execute :sudo, 'apt-get', '-o', 'DPkg::Lock::Timeout=300', 'update'
+        execute :sudo, 'DEBIAN_FRONTEND=noninteractive', 'apt-get',
+                '-o', 'DPkg::Lock::Timeout=300', 'install', '-y',
+                "postgresql-#{new_version}", "postgresql-contrib-#{new_version}"
 
         puts "Upgrading cluster from #{current_version} to #{new_version}..."
         execute :sudo, 'pg_upgradecluster', current_version.to_s, 'main'
@@ -99,8 +101,10 @@ namespace :postgres do
       puts "\nUpdating #{primary}..."
       ssh_executor.execute_on_host(primary) do
         execute :sudo, 'systemctl', 'stop', "postgresql@#{current_version}-main"
-        execute :sudo, 'apt-get', 'update'
-        execute :sudo, 'apt-get', 'install', '-y', "postgresql-#{new_version}", "postgresql-contrib-#{new_version}"
+        execute :sudo, 'apt-get', '-o', 'DPkg::Lock::Timeout=300', 'update'
+        execute :sudo, 'DEBIAN_FRONTEND=noninteractive', 'apt-get',
+                '-o', 'DPkg::Lock::Timeout=300', 'install', '-y',
+                "postgresql-#{new_version}", "postgresql-contrib-#{new_version}"
 
         puts "Upgrading cluster from #{current_version} to #{new_version}..."
         execute :sudo, 'pg_upgradecluster', current_version.to_s, 'main'
@@ -156,8 +160,9 @@ namespace :postgres do
 
       puts "\n📋 Step 1: Patch standby #{standby}"
       ssh_executor.execute_on_host(standby) do
-        execute :sudo, 'apt-get', 'update'
-        execute :sudo, 'apt-get', 'install', '--only-upgrade', '-y', "postgresql-#{version}"
+        execute :sudo, 'apt-get', '-o', 'DPkg::Lock::Timeout=300', 'update'
+        execute :sudo, 'DEBIAN_FRONTEND=noninteractive', 'apt-get',
+                '-o', 'DPkg::Lock::Timeout=300', 'install', '--only-upgrade', '-y', "postgresql-#{version}"
         execute :sudo, 'systemctl', 'restart', "postgresql@#{version}-main"
       end
       puts '✓ Standby patched and restarted'
@@ -169,8 +174,9 @@ namespace :postgres do
 
       puts "\n📋 Step 3: Patch old primary #{primary}"
       ssh_executor.execute_on_host(primary) do
-        execute :sudo, 'apt-get', 'update'
-        execute :sudo, 'apt-get', 'install', '--only-upgrade', '-y', "postgresql-#{version}"
+        execute :sudo, 'apt-get', '-o', 'DPkg::Lock::Timeout=300', 'update'
+        execute :sudo, 'DEBIAN_FRONTEND=noninteractive', 'apt-get',
+                '-o', 'DPkg::Lock::Timeout=300', 'install', '--only-upgrade', '-y', "postgresql-#{version}"
         execute :sudo, 'systemctl', 'restart', "postgresql@#{version}-main"
       end
       puts '✓ Old primary patched'
@@ -211,8 +217,9 @@ namespace :postgres do
       puts "\n🔄 Starting in-place upgrade..."
       ssh_executor.execute_on_host(host) do
         execute :sudo, 'systemctl', 'stop', "postgresql@#{current_version}-main"
-        execute :sudo, 'apt-get', 'update'
-        execute :sudo, 'apt-get', 'install', '-y', "postgresql-#{new_version}"
+        execute :sudo, 'apt-get', '-o', 'DPkg::Lock::Timeout=300', 'update'
+        execute :sudo, 'DEBIAN_FRONTEND=noninteractive', 'apt-get',
+                '-o', 'DPkg::Lock::Timeout=300', 'install', '-y', "postgresql-#{new_version}"
 
         execute :sudo, 'pg_upgradecluster', current_version.to_s, 'main'
         execute :sudo, 'pg_dropcluster', '--stop', current_version.to_s, 'main'

data/templates/pgbouncer_follow_primary.sh.erb

@@ -2,17 +2,19 @@
 set -euo pipefail
 
 REPMGR_CONF="<%= repmgr_conf %>"
+REPMGR_DB="<%= repmgr_database %>"
 PGBOUNCER_INI="/etc/pgbouncer/pgbouncer.ini"
 PGBOUNCER_SERVICE="pgbouncer"
 LOG_TAG="pgbouncer-follow-primary"
 
-cluster_csv=$(/usr/bin/sudo -u <%= postgres_user %> env HOME=/var/lib/postgresql \
-  repmgr -f "$REPMGR_CONF" cluster show --csv 2>/dev/null || true)
-if [[ -z "$cluster_csv" ]]; then
+cluster_nodes=$(/usr/bin/sudo -u <%= postgres_user %> env HOME=/var/lib/postgresql \
+  psql -h /var/run/postgresql -d "$REPMGR_DB" -tAF',' \
+  -c "SELECT type, conninfo FROM repmgr.nodes WHERE active = true" 2>/dev/null || true)
+if [[ -z "$cluster_nodes" ]]; then
   exit 0
 fi
 
-primary_conninfo=$(printf "%s\n" "$cluster_csv" | awk -F',' 'NR>1 && tolower($3) ~ /primary/ {print $NF; exit}')
+primary_conninfo=$(printf "%s\n" "$cluster_nodes" | awk -F',' '$1 == "primary" {print $2; exit}')
 if [[ -z "$primary_conninfo" ]]; then
   exit 0
 fi

data/templates/postgres_log_archive.sh.erb

@@ -0,0 +1,57 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+umask 077
+
+ENV_FILE="/etc/active-postgres-log-archive.env"
+if [[ ! -r "$ENV_FILE" ]]; then
+  echo "missing $ENV_FILE" >&2
+  exit 1
+fi
+
+set -a
+source "$ENV_FILE"
+set +a
+
+LOG_DIR="${POSTGRES_LOG_ARCHIVE_LOG_DIR:-/var/log/postgresql}"
+WORK_DIR="${POSTGRES_LOG_ARCHIVE_WORK_DIR:-/var/tmp/active-postgres-log-archive}"
+PREFIX="${POSTGRES_LOG_ARCHIVE_PREFIX:-postgres-logs}"
+NODE="${POSTGRES_LOG_ARCHIVE_NODE:-$(hostname -f 2>/dev/null || hostname)}"
+DATE_PATH="$(date -u +%Y/%m/%d)"
+STAMP="$(date -u +%Y%m%dT%H%M%SZ)"
+LOCK_FILE="/var/run/active-postgres-log-archive.lock"
+
+mkdir -p "$WORK_DIR"
+exec 9>"$LOCK_FILE"
+flock -n 9 || exit 0
+
+shopt -s nullglob
+log_files=("$LOG_DIR"/postgresql-*.log "$LOG_DIR"/repmgr.log)
+if [[ ${#log_files[@]} -eq 0 ]]; then
+  exit 0
+fi
+
+archive="$WORK_DIR/postgresql-logs-${NODE}-${STAMP}.tar.gz"
+cleanup() {
+  rm -f "$archive"
+}
+trap cleanup EXIT
+
+tar_paths=()
+for file in "${log_files[@]}"; do
+  [[ -f "$file" ]] && tar_paths+=("${file#/}")
+done
+
+if [[ ${#tar_paths[@]} -eq 0 ]]; then
+  exit 0
+fi
+
+tar -czf "$archive" -C / "${tar_paths[@]}"
+
+destination="s3://${AWS_BUCKET}/${PREFIX%/}/${NODE}/${DATE_PATH}/$(basename "$archive")"
+aws_args=(s3 cp "$archive" "$destination" --only-show-errors)
+if [[ -n "${AWS_ENDPOINT_URL:-}" ]]; then
+  aws_args+=(--endpoint-url "$AWS_ENDPOINT_URL")
+fi
+
+aws "${aws_args[@]}"

data/templates/postgresql.conf.erb

@@ -95,8 +95,12 @@ max_parallel_maintenance_workers = <%= pg_config[:max_parallel_maintenance_worke
 logging_collector = <%= pg_config[:logging_collector] || 'on' %>
 log_directory = '<%= pg_config[:log_directory] || 'log' %>'
 log_filename = '<%= pg_config[:log_filename] || 'postgresql-%Y-%m-%d_%H%M%S.log' %>'
+log_file_mode = <%= pg_config[:log_file_mode] || '0600' %>
 log_rotation_age = <%= pg_config[:log_rotation_age] || '1d' %>
 log_rotation_size = <%= pg_config[:log_rotation_size] || '100MB' %>
+<% if pg_config[:log_truncate_on_rotation] %>
+log_truncate_on_rotation = <%= pg_config[:log_truncate_on_rotation] %>
+<% end %>
 log_line_prefix = '%t [%p]: [%l-1] user=%u,db=%d,app=%a,client=%h '
 log_timezone = 'UTC'
 <% if pg_config[:log_checkpoints] %>
@@ -143,7 +147,7 @@ jit = <%= pg_config[:jit] %>
 <% end %>
 
 # Additional settings
-shared_preload_libraries = '<%= pg_config[:shared_preload_libraries] || 'pg_stat_statements' %>'
+shared_preload_libraries = '<%= pg_config[:shared_preload_libraries] || (config.component_enabled?(:repmgr) ? 'pg_stat_statements, repmgr' : 'pg_stat_statements') %>'
 <% if pg_config[:'pg_stat_statements.max'] %>
 pg_stat_statements.max = <%= pg_config[:'pg_stat_statements.max'] %>
 <% end %>
@@ -157,5 +161,3 @@ pg_stat_statements.track = <%= pg_config[:'pg_stat_statements.track'] %>
 <%= key %> = <%= value %>
 <% end %>
 <% end %>
-
-

data/templates/repmgr.conf.erb

@@ -36,10 +36,7 @@ service_stop_command='sudo systemctl stop postgresql@<%= config.version %>-main'
 service_restart_command='sudo systemctl restart postgresql@<%= config.version %>-main'
 service_reload_command='sudo systemctl reload postgresql@<%= config.version %>-main'
 
-ssh_options='-i /var/lib/postgresql/.ssh/active_postgres_dns -o StrictHostKeyChecking=no'
-<% if repmgr_config.key?(:use_rewind) %>
-use_rewind=<%= repmgr_config[:use_rewind] ? 'yes' : 'no' %>
-<% end %>
+ssh_options='-l <%= repmgr_config[:ssh_user] || config.postgres_user %> -i /var/lib/postgresql/.ssh/active_postgres_dns -o IdentitiesOnly=yes -o StrictHostKeyChecking=no'
 
 log_level=INFO
 log_facility=STDERR

data/templates/repmgr_dns_failover.sh.erb

@@ -6,6 +6,7 @@ DNS_USER="<%= dns_user %>"
 DNS_SERVERS=(<%= dns_servers.map(&:to_s).join(' ') %>)
 DNS_SSH_KEY="<%= dns_ssh_key_path %>"
 DNSMASQ_FILE="/etc/dnsmasq.d/active_postgres.conf"
+DNSMASQ_STATIC_FILE="/etc/dnsmasq.d/messhy.conf"
 PRIMARY_RECORDS=(<%= primary_records.map(&:to_s).join(' ') %>)
 REPLICA_RECORDS=(<%= replica_records.map(&:to_s).join(' ') %>)
 SSH_STRICT_HOST_KEY="<%= ssh_strict_host_key %>"
@@ -16,7 +17,7 @@ REPMGR_DB=$(grep -oP "dbname=\K[^ ']+" "$REPMGR_CONF" | head -1)
 REPMGR_USER=$(grep -oP "user=\K[^ ']+" "$REPMGR_CONF" 2>/dev/null | head -1)
 REPMGR_USER="${REPMGR_USER:-repmgr}"
 
-cluster_data=$(sudo -u postgres psql -h /var/run/postgresql -d "$REPMGR_DB" -tAF',' -c "SELECT type, conninfo FROM repmgr.nodes WHERE active = true" 2>/dev/null || true)
+cluster_data=$(psql -h /var/run/postgresql -d "$REPMGR_DB" -tAF',' -c "SELECT type, conninfo FROM repmgr.nodes WHERE active = true" 2>/dev/null || true)
 if [[ -z "$cluster_data" ]]; then
   exit 0
 fi
@@ -48,14 +49,20 @@ if [[ -n "$standby_hosts" ]]; then
 fi
 
 ssh_opts=(-i "$DNS_SSH_KEY" -o BatchMode=yes -o StrictHostKeyChecking="$SSH_STRICT_HOST_KEY" -o UserKnownHostsFile="$SSH_KNOWN_HOSTS")
+cleanup_cmd=""
+for record in "${PRIMARY_RECORDS[@]}" "${REPLICA_RECORDS[@]}"; do
+  if [[ -n "$record" ]]; then
+    cleanup_cmd="${cleanup_cmd}sudo sed -i '\\#^address=/${record}/#d' ${DNSMASQ_STATIC_FILE} 2>/dev/null || true; "
+  fi
+done
 
 for server in "${DNS_SERVERS[@]}"; do
   if [[ -z "$server" ]]; then
     continue
   fi
 
-  if ! /usr/bin/ssh "${ssh_opts[@]}" "${DNS_USER}@${server}" \
-    "sudo bash -c 'cat > ${DNSMASQ_FILE} << \"EOF\"\\n${content}EOF\\n' && sudo systemctl restart dnsmasq"; then
+  if ! printf "%s" "$content" | /usr/bin/ssh "${ssh_opts[@]}" "${DNS_USER}@${server}" \
+    "${cleanup_cmd}sudo tee ${DNSMASQ_FILE} >/dev/null && sudo systemctl restart dnsmasq"; then
     /usr/bin/logger -t "$LOG_TAG" "Failed updating dnsmasq on ${server}"
   fi
 done

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: active_postgres
 version: !ruby/object:Gem::Version
-  version: 0.9.2
+  version: 0.9.4
 platform: ruby
 authors:
 - BoringCache
@@ -200,6 +200,7 @@ files:
 - templates/pgbouncer-follow-primary.timer.erb
 - templates/pgbouncer.ini.erb
 - templates/pgbouncer_follow_primary.sh.erb
+- templates/postgres_log_archive.sh.erb
 - templates/postgresql.conf.erb
 - templates/repmgr.conf.erb
 - templates/repmgr_dns_failover.sh.erb