active_postgres 0.8.0 → 0.9.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 0266b3f7dabc2d844b30aa6410cfd7c59131a7fb77a2dea85f01f9a66c2a2443
-  data.tar.gz: fd720306229f1ddea1d4e01a9dc3a5df3b7a60a620c38e86779e82f9698ebcbc
+  metadata.gz: 967673143f28952b4f44c54cb97282ca9ca288fe8885fc525433baac00633db0
+  data.tar.gz: a43859d367af55c17c9a6a407a39e2e492cc829806ca615dd4f82b03604303f0
 SHA512:
-  metadata.gz: bb3bc17966f9ebe98a55f09f678fe2bc2e57b92615c45848c347a7568f6e3bb61d15788d6bb16008e4498a9b944a3f53241f2e2579b204d28679341b041f4978
-  data.tar.gz: 3863cace22d1ba3aea901d312d26152de067c7ca898ddff5b654a75702a9e6b4c01f6ffbac820fed338fc5db8ce011ad39676143f81f785f66f7fec72cd72ae5
+  metadata.gz: f6dc9378f1391e20051028100f9767a65a9de62642719b4a413605a9208fa9a25b5e0c6790c9fdd0e966080293e5f33e9a5b9e860c5622cc20b65d933d70dae6
+  data.tar.gz: 17b10e8adba8c2a502d0217f156ce68e7c2d1f4e4442fddd82b3ad8e560a940dc7a07a360b9a662dd8fbf0e8f80af7aff2f37bc6d9aaba7c0195b8590c32fbb3

data/README.md

@@ -7,7 +7,7 @@ Production-grade PostgreSQL HA for Rails.
 - **High Availability**: Primary/standby replication with automatic failover (repmgr)
 - **Connection Pooling**: PgBouncer integration
 - **Rails Integration**: Automatic database.yml config, migration guard, read replica routing
-- **Modular Components**: Core, Performance Tuning, repmgr, PgBouncer, pgBackRest, Monitoring, SSL, Extensions
+- **Modular Components**: Core, Performance Tuning (opt-in), repmgr, PgBouncer, pgBackRest, Monitoring, SSL, Extensions
 
 ## Quick Start
 
@@ -48,6 +48,7 @@ production:
     superuser_password: $POSTGRES_SUPERUSER_PASSWORD
     replication_password: $POSTGRES_REPLICATION_PASSWORD
     repmgr_password: $POSTGRES_REPMGR_PASSWORD
+    monitoring_password: $POSTGRES_MONITORING_PASSWORD
     app_password: $POSTGRES_APP_PASSWORD
 ```
 
@@ -110,14 +111,84 @@ active_postgres cache-secrets
 | Component | Description | Config |
 |-----------|-------------|--------|
 | **Core** | PostgreSQL installation | Always enabled |
-| **Performance Tuning** | Auto-optimization | Enabled by default |
+| **Performance Tuning** | Auto-optimization | Disabled by default |
 | **repmgr** | HA & automatic failover | `repmgr: {enabled: true}` |
 | **PgBouncer** | Connection pooling | `pgbouncer: {enabled: true}` |
 | **pgBackRest** | Backup & restore | `pgbackrest: {enabled: true}` |
-| **Monitoring** | postgres_exporter | `monitoring: {enabled: true}` |
+| **Monitoring** | postgres_exporter (configures a dedicated pg_monitor user) | `monitoring: {enabled: true}` |
 | **SSL** | Encrypted connections | `ssl: {enabled: true}` |
 | **Extensions** | pgvector, PostGIS, etc. | `extensions: {enabled: true, list: [pgvector]}` |
 
+### Monitoring credentials
+
+`postgres_exporter` uses a dedicated `pg_monitor` user. Provide a password in secrets and optionally set the username:
+
+```yaml
+components:
+  monitoring: {enabled: true, user: postgres_exporter}
+secrets:
+  monitoring_password: $POSTGRES_MONITORING_PASSWORD
+```
+
+### pgBackRest S3-compatible endpoints
+
+For Tigris/MinIO/Wasabi/DO Spaces, set a custom endpoint and use path-style URLs:
+
+```yaml
+components:
+  pgbackrest:
+    enabled: true
+    repo_type: s3
+    s3_bucket: myapp-backups
+    s3_region: auto
+    s3_endpoint: t3.storage.dev
+    s3_uri_style: path
+```
+
+### Stable app endpoint with PgBouncer
+
+If you run PgBouncer on each PostgreSQL node and want a fixed app URL, enable `follow_primary`.
+Each PgBouncer instance periodically repoints to the current primary using repmgr metadata.
+Put a TCP load balancer or DNS record in front of the PgBouncer nodes.
+
+```yaml
+components:
+  pgbouncer:
+    enabled: true
+    follow_primary: true
+    follow_primary_interval: 5
+```
+
+### Automatic DNS updates on failover (dnsmasq)
+
+If you use Messhy’s mesh DNS (dnsmasq), repmgr can update writer/reader DNS records on failover.
+Enable `dns_failover` under `repmgr` and provide DNS server IPs or hostnames.
+If you run setup from outside the mesh, use objects with both the public SSH host
+and the private WireGuard IP so setup can SSH in and failover updates stay on the mesh.
+If you run setup from inside the mesh, you can use a simple list of private IPs.
+
+```yaml
+components:
+  repmgr:
+    enabled: true
+    dns_failover:
+      enabled: true
+      provider: dnsmasq
+      domain: mesh.internal
+      dns_servers:
+        - host: 18.170.173.14
+          private_ip: 10.8.0.10
+        - host: 98.85.183.175
+          private_ip: 10.8.0.110
+      primary_record: db-primary.mesh.internal
+      replica_record: db-replica.mesh.internal
+```
+
+This installs an event hook so repmgr updates `/etc/dnsmasq.d/active_postgres.conf`
+on the DNS servers whenever the primary changes.
+DNS servers must be reachable over the private network and allow SSH from the
+database nodes (active_postgres installs SSH keys on first setup).
+
 ## Secrets Management
 
 ```yaml
@@ -127,7 +198,8 @@ secrets:
 
   # Command execution
   password: $(op read "op://vault/item/field")
-  password: $(rails runner "puts Rails.application.credentials.dig(:postgres, :password)")
+  password: rails_credentials:postgres.password
+  password: credentials:postgres.password
 
   # AWS Secrets Manager
   password: $(aws secretsmanager get-secret-value --secret-id myapp/postgres --query SecretString)
@@ -150,9 +222,19 @@ User.all                     # → Replica
 - Ruby 3.0+
 - PostgreSQL 12+
 - Ubuntu 20.04+ / Debian 11+ with systemd
-- SSH key-based authentication
+- SSH key-based authentication (hosts must be in `~/.ssh/known_hosts`)
 - Rails 6.0+ (optional)
 
+### SSH host key verification
+
+By default, host keys are verified strictly (`always`). For first-time provisioning you can set:
+
+```yaml
+ssh_host_key_verification: accept_new
+```
+
+This trusts a host key the first time and then pins it; prefer `always` once hosts are known.
+
 ## License
 
 MIT

data/lib/active_postgres/components/core.rb

@@ -189,14 +189,9 @@ module ActivePostgres
         app_password = resolve_app_password
         sql = build_app_user_sql(app_user, app_database, app_password)
 
-        ssh_executor.execute_on_host(host) do
-          upload! StringIO.new(sql), '/tmp/create_app_user.sql'
-          execute :chmod, '644', '/tmp/create_app_user.sql'
-          execute :sudo, '-u', 'postgres', 'psql', '-f', '/tmp/create_app_user.sql'
-          execute :rm, '-f', '/tmp/create_app_user.sql'
+        ssh_executor.run_sql(host, sql, postgres_user: 'postgres', tuples_only: false, capture: false)
 
-          puts "  ✓ Created app user '#{app_user}' and database '#{app_database}'"
-        end
+        puts "  ✓ Created app user '#{app_user}' and database '#{app_database}'"
       rescue StandardError => e
         warn "  Warning: Could not create app user: #{e.message}"
         warn '  You may need to create the user manually'

data/lib/active_postgres/components/extensions.rb

@@ -33,17 +33,27 @@ module ActivePostgres
         puts 'Extensions uninstall not implemented (extensions remain in database)'
       end
 
-      private
+      def restart
+        puts 'Extensions do not require restart (loaded at database connection time)'
+      end
 
-      def install_on_primary(extensions)
-        host = config.primary_host
-        version = config.version
-        db_name = config.secrets_config['database_name'] || 'postgres'
-        postgres_user = config.postgres_user
+      def install_on_standby(standby_host)
+        extensions_config = config.component_config(:extensions)
+        return unless extensions_config[:enabled]
 
-        puts "  Installing extensions on primary (#{host})..."
+        extensions = extensions_config[:list] || []
+        return if extensions.empty?
+
+        puts "Installing extension packages on standby #{standby_host}..."
+        install_packages_on_host(standby_host, extensions)
+      end
+
+      private
 
+      def install_packages_on_host(host, extensions)
+        version = config.version
         packages_to_install = []
+
         extensions.each do |ext_name|
           package = EXTENSION_PACKAGES[ext_name]
           next unless package
@@ -52,46 +62,41 @@ module ActivePostgres
           packages_to_install << package
         end
 
+        return if packages_to_install.empty?
+
         ssh_executor.execute_on_host(host) do
-          unless packages_to_install.empty?
-            execute :sudo, 'apt-get', 'update', '-qq'
-            execute :sudo, 'DEBIAN_FRONTEND=noninteractive', 'apt-get', 'install', '-y', '-qq',
-                    *packages_to_install
-          end
+          execute :sudo, 'apt-get', 'update', '-qq'
+          execute :sudo, 'DEBIAN_FRONTEND=noninteractive', 'apt-get', 'install', '-y', '-qq',
+                  *packages_to_install
+        end
+      end
+
+      def install_on_primary(extensions)
+        host = config.primary_host
+        db_name = config.secrets_config['database_name'] || 'postgres'
+        postgres_user = config.postgres_user
+
+        puts "  Installing extensions on primary (#{host})..."
+
+        install_packages_on_host(host, extensions)
 
+        ssh_executor.execute_on_host(host) do
           extensions.each do |ext_name|
-            sql = "CREATE EXTENSION IF NOT EXISTS #{ext_name};"
+            sql = "CREATE EXTENSION IF NOT EXISTS \"#{ext_name}\";"
             begin
               execute :sudo, '-u', postgres_user, 'psql', '-d', db_name, '-c', sql
-            rescue StandardError
-              nil
+              info "✓ Extension #{ext_name} created/verified"
+            rescue StandardError => e
+              warn "⚠ Could not create extension #{ext_name}: #{e.message}"
             end
           end
         end
       end
 
       def install_on_standbys(extensions)
-        version = config.version
-
         config.standby_hosts.each do |host|
-          puts "  Installing extensions on standby (#{host})..."
-
-          packages_to_install = []
-          extensions.each do |ext_name|
-            package = EXTENSION_PACKAGES[ext_name]
-            next unless package
-
-            package = package.gsub('{version}', version.to_s)
-            packages_to_install << package
-          end
-
-          ssh_executor.execute_on_host(host) do
-            unless packages_to_install.empty?
-              execute :sudo, 'apt-get', 'update', '-qq'
-              execute :sudo, 'DEBIAN_FRONTEND=noninteractive', 'apt-get', 'install', '-y', '-qq',
-                      *packages_to_install
-            end
-          end
+          puts "  Installing extension packages on standby (#{host})..."
+          install_packages_on_host(host, extensions)
         end
       end
     end

data/lib/active_postgres/components/monitoring.rb

@@ -1,9 +1,13 @@
+require 'cgi'
+
 module ActivePostgres
   module Components
     class Monitoring < Base
       def install
         puts 'Installing postgres_exporter for monitoring...'
 
+        ensure_monitoring_user
+
         config.all_hosts.each do |host|
           install_on_host(host)
         end
@@ -14,8 +18,9 @@ module ActivePostgres
 
         config.all_hosts.each do |host|
           ssh_executor.execute_on_host(host) do
-            execute :sudo, 'systemctl', 'stop', 'postgres_exporter'
-            execute :sudo, 'systemctl', 'disable', 'postgres_exporter'
+            execute :sudo, 'systemctl', 'stop', 'prometheus-postgres-exporter'
+            execute :sudo, 'systemctl', 'disable', 'prometheus-postgres-exporter'
+            execute :sudo, 'apt-get', 'remove', '-y', 'prometheus-postgres-exporter'
           end
         end
       end
@@ -25,11 +30,16 @@ module ActivePostgres
 
         config.all_hosts.each do |host|
           ssh_executor.execute_on_host(host) do
-            execute :sudo, 'systemctl', 'restart', 'postgres_exporter'
+            execute :sudo, 'systemctl', 'restart', 'prometheus-postgres-exporter'
           end
         end
       end
 
+      def install_on_standby(standby_host)
+        puts "Installing postgres_exporter on standby #{standby_host}..."
+        install_on_host(standby_host)
+      end
+
       private
 
       def install_on_host(host)
@@ -42,14 +52,91 @@ module ActivePostgres
         ssh_executor.execute_on_host(host) do
           # Install via package manager or download binary
           execute :sudo, 'apt-get', 'install', '-y', '-qq', 'prometheus-postgres-exporter'
+        end
+
+        configure_exporter_service(host, monitoring_config, exporter_port)
 
+        ssh_executor.execute_on_host(host) do
           # Enable and start
           execute :sudo, 'systemctl', 'enable', 'prometheus-postgres-exporter'
-          execute :sudo, 'systemctl', 'start', 'prometheus-postgres-exporter'
+          execute :sudo, 'systemctl', 'restart', 'prometheus-postgres-exporter'
         end
 
         puts "  Metrics available at: http://#{host}:#{exporter_port}/metrics"
       end
+
+      def ensure_monitoring_user
+        monitoring_config = config.component_config(:monitoring)
+        monitoring_user = monitoring_config[:user] || 'postgres_exporter'
+        monitoring_password = normalize_monitoring_password(secrets.resolve('monitoring_password'))
+
+        sql = build_monitoring_user_sql(monitoring_user, monitoring_password)
+
+        ssh_executor.run_sql(config.primary_host, sql, postgres_user: 'postgres', port: 5432, tuples_only: false,
+                                                     capture: false)
+      end
+
+      def build_monitoring_user_sql(user, password)
+        escaped_password = password.gsub("'", "''")
+
+        [
+          'DO $$',
+          'BEGIN',
+          "  IF NOT EXISTS (SELECT FROM pg_roles WHERE rolname = '#{user}') THEN",
+          "    CREATE USER #{user} WITH LOGIN PASSWORD '#{escaped_password}';",
+          '  ELSE',
+          "    ALTER USER #{user} WITH LOGIN PASSWORD '#{escaped_password}';",
+          '  END IF;',
+          'END $$;',
+          '',
+          "GRANT pg_monitor TO #{user};",
+          ''
+        ].join("\n")
+      end
+
+      def configure_exporter_service(host, monitoring_config, exporter_port)
+        monitoring_user = monitoring_config[:user] || 'postgres_exporter'
+        monitoring_password = normalize_monitoring_password(secrets.resolve('monitoring_password'))
+
+        dsn = build_exporter_dsn(monitoring_config, monitoring_user, monitoring_password)
+        override = <<~CONF
+          [Service]
+          Environment="DATA_SOURCE_NAME=#{escape_systemd_env(dsn)}"
+          Environment="PG_EXPORTER_WEB_LISTEN_ADDRESS=:#{exporter_port}"
+        CONF
+
+        ssh_executor.execute_on_host(host) do
+          execute :sudo, 'mkdir', '-p', '/etc/systemd/system/prometheus-postgres-exporter.service.d'
+          upload! StringIO.new(override), '/tmp/prometheus-postgres-exporter.override'
+          execute :sudo, 'mv', '/tmp/prometheus-postgres-exporter.override',
+                  '/etc/systemd/system/prometheus-postgres-exporter.service.d/override.conf'
+          execute :sudo, 'chown', 'root:root', '/etc/systemd/system/prometheus-postgres-exporter.service.d/override.conf'
+          execute :sudo, 'chmod', '600', '/etc/systemd/system/prometheus-postgres-exporter.service.d/override.conf'
+          execute :sudo, 'systemctl', 'daemon-reload'
+        end
+      end
+
+      def build_exporter_dsn(monitoring_config, monitoring_user, monitoring_password)
+        host = monitoring_config[:database_host] || 'localhost'
+        port = monitoring_config[:database_port] || 5432
+        database = monitoring_config[:database] || 'postgres'
+        sslmode = config.component_enabled?(:ssl) ? 'require' : 'prefer'
+
+        encoded_password = CGI.escape(monitoring_password.to_s)
+        "postgresql://#{monitoring_user}:#{encoded_password}@#{host}:#{port}/#{database}?sslmode=#{sslmode}"
+      end
+
+      def normalize_monitoring_password(raw_password)
+        password = raw_password.to_s.rstrip
+
+        raise Error, 'monitoring_password secret is missing (required for monitoring)' if password.empty?
+
+        password
+      end
+
+      def escape_systemd_env(value)
+        value.to_s.gsub('\\', '\\\\').gsub('"', '\\"')
+      end
     end
   end
 end

data/lib/active_postgres/components/pgbackrest.rb

@@ -19,6 +19,7 @@ module ActivePostgres
         config.all_hosts.each do |host|
           ssh_executor.execute_on_host(host) do
             execute :sudo, 'apt-get', 'remove', '-y', 'pgbackrest'
+            execute :sudo, 'rm', '-f', '/etc/cron.d/pgbackrest-backup'
           end
         end
       end
@@ -27,6 +28,11 @@ module ActivePostgres
         puts "pgBackRest is a backup tool and doesn't run as a service."
       end
 
+      def install_on_standby(host)
+        puts "  Installing pgBackRest on standby #{host}..."
+        install_on_host(host, create_stanza: false)
+      end
+
       def run_backup(type = 'full')
         puts "Running #{type} backup..."
         postgres_user = config.postgres_user
@@ -66,7 +72,7 @@ module ActivePostgres
       def install_on_host(host, create_stanza: true)
         puts "  Installing pgBackRest on #{host}..."
 
-        pgbackrest_config = config.component_config(:pgbackrest)
+        pgbackrest_config = secrets.resolve_value(config.component_config(:pgbackrest))
         postgres_user = config.postgres_user
         secrets_obj = secrets
         _ = pgbackrest_config # Used in ERB template
@@ -78,7 +84,8 @@ module ActivePostgres
         end
 
         # Upload configuration
-        upload_template(host, 'pgbackrest.conf.erb', '/etc/pgbackrest.conf', binding, mode: '644')
+        upload_template(host, 'pgbackrest.conf.erb', '/etc/pgbackrest.conf', binding,
+                        mode: '640', owner: "root:#{postgres_user}")
 
         ssh_executor.execute_on_host(host) do
           execute :sudo, 'rm', '-rf', '/var/lib/pgbackrest', '||', 'true'
@@ -99,6 +106,35 @@ module ActivePostgres
             execute :sudo, '-u', postgres_user, 'pgbackrest', '--stanza=main', 'stanza-create'
           end
         end
+
+        # Set up scheduled backups on primary only
+        if create_stanza && pgbackrest_config[:schedule]
+          setup_backup_schedule(host, pgbackrest_config[:schedule])
+        end
+      end
+
+      def setup_backup_schedule(host, schedule)
+        puts "  Setting up backup schedule: #{schedule}"
+        postgres_user = config.postgres_user
+
+        # Create cron job for scheduled backups
+        # /etc/cron.d format requires username after time spec
+        cron_content = <<~CRON
+          # pgBackRest scheduled backups (managed by active_postgres)
+          SHELL=/bin/bash
+          PATH=/usr/local/sbin:/usr/local/bin:/sbin:/bin:/usr/sbin:/usr/bin
+          #{schedule} #{postgres_user} pgbackrest --stanza=main --type=full backup
+        CRON
+
+        # Install cron job in /etc/cron.d (system cron directory)
+        # Use a temp file + sudo mv to avoid shell redirection permission issues.
+        ssh_executor.upload_file(host, cron_content, '/etc/cron.d/pgbackrest-backup', mode: '644', owner: 'root:root')
+      end
+
+      def remove_backup_schedule(host)
+        ssh_executor.execute_on_host(host) do
+          execute :sudo, 'rm', '-f', '/etc/cron.d/pgbackrest-backup'
+        end
       end
     end
   end

data/lib/active_postgres/components/pgbouncer.rb

@@ -14,6 +14,11 @@ module ActivePostgres
 
         config.all_hosts.each do |host|
           ssh_executor.execute_on_host(host) do
+            execute :sudo, 'systemctl', 'disable', '--now', 'pgbouncer-follow-primary.timer', '||', 'true'
+            execute :sudo, 'rm', '-f', '/usr/local/bin/pgbouncer-follow-primary',
+                    '/etc/systemd/system/pgbouncer-follow-primary.service',
+                    '/etc/systemd/system/pgbouncer-follow-primary.timer'
+            execute :sudo, 'systemctl', 'daemon-reload'
             execute :sudo, 'systemctl', 'stop', 'pgbouncer'
             execute :sudo, 'apt-get', 'remove', '-y', 'pgbouncer'
           end
@@ -53,12 +58,22 @@ module ActivePostgres
         puts "  Installing PgBouncer on #{host}..."
 
         user_config = config.component_config(:pgbouncer)
+        follow_primary = user_config[:follow_primary] == true
+        if follow_primary && !config.component_enabled?(:repmgr)
+          raise Error, 'PgBouncer follow_primary requires repmgr to be enabled'
+        end
 
         max_connections = get_postgres_max_connections(host)
         optimal_pool = ConnectionPooler.calculate_optimal_pool_sizes(max_connections)
 
         pgbouncer_config = optimal_pool.merge(user_config)
+        pgbouncer_config[:database_host] = config.primary_replication_host if follow_primary
         ssl_enabled = config.component_enabled?(:ssl)
+        has_ca_cert = ssl_enabled && secrets.resolve('ssl_chain')
+        secrets_obj = secrets
+        _ = pgbouncer_config
+        _ = has_ca_cert
+        _ = secrets_obj
 
         puts "  Calculated pool settings for max_connections=#{max_connections}"
 
@@ -76,6 +91,8 @@ module ActivePostgres
           execute :sudo, 'systemctl', 'enable', 'pgbouncer'
           execute :sudo, 'systemctl', 'restart', 'pgbouncer'
         end
+
+        install_follow_primary(host, pgbouncer_config) if follow_primary
       end
 
       def setup_ssl_certs(host)
@@ -130,10 +147,7 @@ module ActivePostgres
       def fetch_user_hash(backend, user, postgres_user)
         sql = build_user_hash_sql(user)
 
-        backend.upload! StringIO.new(sql), '/tmp/get_user_hash.sql'
-        backend.execute :chmod, '644', '/tmp/get_user_hash.sql'
-        user_hash = backend.capture(:sudo, '-u', postgres_user, 'psql', '-t', '-f', '/tmp/get_user_hash.sql').strip
-        backend.execute :rm, '-f', '/tmp/get_user_hash.sql'
+        user_hash = ssh_executor.run_sql_on_backend(backend, sql, postgres_user: postgres_user).to_s.strip
 
         if user_hash && !user_hash.empty?
           puts "  ✓ Added #{user} to PgBouncer userlist"
@@ -167,6 +181,31 @@ module ActivePostgres
           warn '  Warning: No users added to userlist - connections may fail'
         end
       end
+
+      def install_follow_primary(host, pgbouncer_config)
+        interval = pgbouncer_config[:follow_primary_interval] || 5
+        interval = interval.to_i
+        interval = 5 if interval <= 0
+
+        repmgr_conf = '/etc/repmgr.conf'
+        postgres_user = config.postgres_user
+        _ = interval
+        _ = repmgr_conf
+        _ = postgres_user
+
+        upload_template(host, 'pgbouncer_follow_primary.sh.erb', '/usr/local/bin/pgbouncer-follow-primary', binding,
+                        mode: '755', owner: 'root:root')
+        upload_template(host, 'pgbouncer-follow-primary.service.erb',
+                        '/etc/systemd/system/pgbouncer-follow-primary.service', binding, mode: '644', owner: 'root:root')
+        upload_template(host, 'pgbouncer-follow-primary.timer.erb',
+                        '/etc/systemd/system/pgbouncer-follow-primary.timer', binding, mode: '644', owner: 'root:root')
+
+        ssh_executor.execute_on_host(host) do
+          execute :sudo, 'systemctl', 'daemon-reload'
+          execute :sudo, 'systemctl', 'enable', '--now', 'pgbouncer-follow-primary.timer'
+          execute :sudo, 'systemctl', 'start', 'pgbouncer-follow-primary.service'
+        end
+      end
     end
   end
 end