active_postgres 0.9.0 → 0.9.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 967673143f28952b4f44c54cb97282ca9ca288fe8885fc525433baac00633db0
-  data.tar.gz: a43859d367af55c17c9a6a407a39e2e492cc829806ca615dd4f82b03604303f0
+  metadata.gz: e51d5f1214cedf497a9b060e5c667c43711f9ff5e36024bfd8dc926b50fe272f
+  data.tar.gz: 618fa70d39a43e1c74ba2ed449b75138670bca92cb6bf418606f772de34c6a7d
 SHA512:
-  metadata.gz: f6dc9378f1391e20051028100f9767a65a9de62642719b4a413605a9208fa9a25b5e0c6790c9fdd0e966080293e5f33e9a5b9e860c5622cc20b65d933d70dae6
-  data.tar.gz: 17b10e8adba8c2a502d0217f156ce68e7c2d1f4e4442fddd82b3ad8e560a940dc7a07a360b9a662dd8fbf0e8f80af7aff2f37bc6d9aaba7c0195b8590c32fbb3
+  metadata.gz: 254b6fb4342d4e510076d05ebfbeaa0e7990b463df13ad6ddb7d6eb5a70562c23cc4f502ce6c435c6372fc3dd5f695285ee3fdfed522cc188f1b1cc21c6cc90d
+  data.tar.gz: d9147a4679529344ccb0ffd3d5c2ffacc3072464b9cf4e1c6e11150a66c142ef0d0b45f9b9cd9d34eceba6f4d0e50baa7c2f227a5b2b1719502972e2b4b39ee4

data/README.md

@@ -69,6 +69,8 @@ postgres:
 ```bash
 rake postgres:setup   # Deploy cluster
 rake postgres:status  # Check health
+rake postgres:overview  # Control tower overview
+rake postgres:help    # Task summary
 ```
 
 ## Common Operations
@@ -83,8 +85,10 @@ rake postgres:promote[host]            # Promote standby to primary
 
 # Backups (requires pgBackRest)
 rake postgres:backup:full
+rake postgres:backup:incremental
 rake postgres:backup:list
 rake postgres:backup:restore[backup_id]
+rake postgres:backup:restore_at["2026-01-29 01:15:00",promote]
 
 # Credential rotation (zero downtime)
 rake postgres:credentials:rotate_random
@@ -101,6 +105,7 @@ rake postgres:update:patch             # Security patches
 active_postgres setup --environment=production
 active_postgres setup-standby HOST
 active_postgres status
+active_postgres overview
 active_postgres promote HOST
 active_postgres backup --type=full
 active_postgres cache-secrets
@@ -115,21 +120,37 @@ active_postgres cache-secrets
 | **repmgr** | HA & automatic failover | `repmgr: {enabled: true}` |
 | **PgBouncer** | Connection pooling | `pgbouncer: {enabled: true}` |
 | **pgBackRest** | Backup & restore | `pgbackrest: {enabled: true}` |
-| **Monitoring** | postgres_exporter (configures a dedicated pg_monitor user) | `monitoring: {enabled: true}` |
+| **Monitoring** | postgres_exporter + optional node_exporter/Grafana | `monitoring: {enabled: true}` |
 | **SSL** | Encrypted connections | `ssl: {enabled: true}` |
 | **Extensions** | pgvector, PostGIS, etc. | `extensions: {enabled: true, list: [pgvector]}` |
 
 ### Monitoring credentials
 
-`postgres_exporter` uses a dedicated `pg_monitor` user. Provide a password in secrets and optionally set the username:
+`postgres_exporter` uses a dedicated `pg_monitor` user. Provide a password in secrets and optionally set the username.
+Enable node_exporter for host-level metrics, and Grafana if you want a local UI:
 
 ```yaml
 components:
-  monitoring: {enabled: true, user: postgres_exporter}
+  monitoring:
+    enabled: true
+    user: postgres_exporter
+    node_exporter: true
+    node_exporter_port: 9100
+    # node_exporter_listen_address: 10.8.0.10
+    # grafana:
+    #   enabled: true
+    #   host: grafana.example.com
+    #   listen_address: 10.8.0.10
+    #   port: 3000
+    #   prometheus_url: http://prometheus.example.com:9090
 secrets:
   monitoring_password: $POSTGRES_MONITORING_PASSWORD
+  # grafana_admin_password: $GRAFANA_ADMIN_PASSWORD
 ```
 
+When Grafana is enabled, `grafana_admin_password` is required and Grafana will be installed
+on the configured host. If `prometheus_url` is provided, a Prometheus datasource is provisioned.
+
 ### pgBackRest S3-compatible endpoints
 
 For Tigris/MinIO/Wasabi/DO Spaces, set a custom endpoint and use path-style URLs:
@@ -145,6 +166,35 @@ components:
     s3_uri_style: path
 ```
 
+### Point-in-time recovery (PITR)
+
+Ensure WAL retention is long enough for your recovery window:
+
+```yaml
+components:
+  pgbackrest:
+    retention_full: 7
+    retention_archive: 14
+    # You can schedule full + incremental separately:
+    # schedule_full: "0 2 * * *"
+    # schedule_incremental: "0 * * * *"
+```
+
+Run a PITR restore:
+
+```bash
+rake postgres:backup:restore_at["2026-01-29 01:15:00",promote]
+```
+
+If you want the old primary to rejoin without a full re-clone after failover,
+enable pg_rewind support in repmgr:
+
+```yaml
+components:
+  repmgr:
+    use_rewind: true
+```
+
 ### Stable app endpoint with PgBouncer
 
 If you run PgBouncer on each PostgreSQL node and want a fixed app URL, enable `follow_primary`.
@@ -175,6 +225,8 @@ components:
       enabled: true
       provider: dnsmasq
       domain: mesh.internal
+      # Or use multiple domains during cutover:
+      # domains: [mesh.internal, mesh.v2.internal]
       dns_servers:
         - host: 18.170.173.14
           private_ip: 10.8.0.10
@@ -182,6 +234,9 @@ components:
           private_ip: 10.8.0.110
       primary_record: db-primary.mesh.internal
       replica_record: db-replica.mesh.internal
+      # Or multiple records explicitly:
+      # primary_records: [db-primary.mesh.internal, db-primary.mesh.v2.internal]
+      # replica_records: [db-replica.mesh.internal, db-replica.mesh.v2.internal]
 ```
 
 This installs an event hook so repmgr updates `/etc/dnsmasq.d/active_postgres.conf`

data/lib/active_postgres/cli.rb

@@ -49,6 +49,13 @@ module ActivePostgres
       health_checker.show_status
     end
 
+    desc 'overview', 'Show control tower overview'
+    def overview
+      config = load_config
+      overview = Overview.new(config)
+      overview.show
+    end
+
     desc 'health', 'Run health checks'
     def health
       config = load_config

data/lib/active_postgres/components/core.rb

@@ -49,6 +49,11 @@ module ActivePostgres
         create_app_user_and_database(config.primary_host)
       end
 
+      def install_packages_only(host)
+        puts "  Installing packages on #{host} (cluster will be created by repmgr)..."
+        ssh_executor.install_postgres(host, config.version)
+      end
+
       private
 
       def install_on_host(host, is_primary:)
@@ -96,11 +101,6 @@ module ActivePostgres
         optimal_settings.merge(user_postgresql)
       end
 
-      def install_packages_only(host)
-        puts "  Installing packages on #{host} (cluster will be created by repmgr)..."
-        ssh_executor.install_postgres(host, config.version)
-      end
-
       def cluster_exists?(host)
         exists = false
         version = config.version

data/lib/active_postgres/components/monitoring.rb

@@ -11,6 +11,8 @@ module ActivePostgres
         config.all_hosts.each do |host|
           install_on_host(host)
         end
+
+        install_grafana_if_enabled
       end
 
       def uninstall
@@ -21,8 +23,16 @@ module ActivePostgres
             execute :sudo, 'systemctl', 'stop', 'prometheus-postgres-exporter'
             execute :sudo, 'systemctl', 'disable', 'prometheus-postgres-exporter'
             execute :sudo, 'apt-get', 'remove', '-y', 'prometheus-postgres-exporter'
+
+            if node_exporter_enabled?
+              execute :sudo, 'systemctl', 'stop', 'prometheus-node-exporter'
+              execute :sudo, 'systemctl', 'disable', 'prometheus-node-exporter'
+              execute :sudo, 'apt-get', 'remove', '-y', 'prometheus-node-exporter'
+            end
           end
         end
+
+        uninstall_grafana_if_enabled
       end
 
       def restart
@@ -31,6 +41,7 @@ module ActivePostgres
         config.all_hosts.each do |host|
           ssh_executor.execute_on_host(host) do
             execute :sudo, 'systemctl', 'restart', 'prometheus-postgres-exporter'
+            execute :sudo, 'systemctl', 'restart', 'prometheus-node-exporter' if node_exporter_enabled?
           end
         end
       end
@@ -63,6 +74,8 @@ module ActivePostgres
         end
 
         puts "  Metrics available at: http://#{host}:#{exporter_port}/metrics"
+
+        install_node_exporter(host, monitoring_config) if node_exporter_enabled?
       end
 
       def ensure_monitoring_user
@@ -134,6 +147,158 @@ module ActivePostgres
         password
       end
 
+      def node_exporter_enabled?
+        monitoring_config = config.component_config(:monitoring)
+        monitoring_config.fetch(:node_exporter, false) == true
+      end
+
+      def install_node_exporter(host, monitoring_config)
+        port = monitoring_config[:node_exporter_port] || 9100
+        listen_address = monitoring_config[:node_exporter_listen_address].to_s.strip
+        puts "  Installing node_exporter on #{host}..."
+
+        ssh_executor.execute_on_host(host) do
+          execute :sudo, 'apt-get', 'install', '-y', '-qq', 'prometheus-node-exporter'
+        end
+
+        configure_node_exporter_service(host, port, listen_address)
+
+        ssh_executor.execute_on_host(host) do
+          execute :sudo, 'systemctl', 'enable', 'prometheus-node-exporter'
+          execute :sudo, 'systemctl', 'restart', 'prometheus-node-exporter'
+        end
+
+        puts "  Node metrics available at: http://#{host}:#{port}/metrics"
+      end
+
+      def configure_node_exporter_service(host, port, listen_address)
+        listen = if listen_address.empty?
+                   ":#{port}"
+                 else
+                   "#{listen_address}:#{port}"
+                 end
+
+        return if listen == ':9100'
+
+        override = <<~CONF
+          [Service]
+          ExecStart=
+          ExecStart=/usr/bin/prometheus-node-exporter --web.listen-address=#{listen}
+        CONF
+
+        ssh_executor.execute_on_host(host) do
+          execute :sudo, 'mkdir', '-p', '/etc/systemd/system/prometheus-node-exporter.service.d'
+          upload! StringIO.new(override), '/tmp/prometheus-node-exporter.override'
+          execute :sudo, 'mv', '/tmp/prometheus-node-exporter.override',
+                  '/etc/systemd/system/prometheus-node-exporter.service.d/override.conf'
+          execute :sudo, 'chown', 'root:root', '/etc/systemd/system/prometheus-node-exporter.service.d/override.conf'
+          execute :sudo, 'chmod', '644', '/etc/systemd/system/prometheus-node-exporter.service.d/override.conf'
+          execute :sudo, 'systemctl', 'daemon-reload'
+        end
+      end
+
+      def install_grafana_if_enabled
+        monitoring_config = config.component_config(:monitoring)
+        grafana_config = monitoring_config[:grafana] || {}
+        return unless grafana_config[:enabled]
+
+        host = grafana_config[:host].to_s.strip
+        raise Error, 'monitoring.grafana.host is required when grafana is enabled' if host.empty?
+
+        admin_password = normalize_grafana_password(secrets.resolve('grafana_admin_password'))
+        prometheus_url = grafana_config[:prometheus_url]
+        listen_address = grafana_config[:listen_address].to_s.strip
+        port = (grafana_config[:port] || 3000).to_i
+
+        puts "Installing Grafana on #{host}..."
+
+        ssh_executor.execute_on_host(host) do
+          execute :sudo, 'apt-get', 'install', '-y', '-qq', 'apt-transport-https', 'software-properties-common', 'wget'
+          execute :sudo, 'wget', '-q', '-O', '/usr/share/keyrings/grafana.gpg', 'https://apt.grafana.com/gpg.key'
+          execute :sudo, 'sh', '-c',
+                  'echo "deb [signed-by=/usr/share/keyrings/grafana.gpg] https://apt.grafana.com stable main" > /etc/apt/sources.list.d/grafana.list'
+          execute :sudo, 'apt-get', 'update', '-qq'
+          execute :sudo, 'apt-get', 'install', '-y', '-qq', 'grafana'
+          execute :sudo, 'systemctl', 'enable', '--now', 'grafana-server'
+          execute :sudo, 'grafana-cli', 'admin', 'reset-admin-password', admin_password
+        end
+
+        configure_grafana_service(host, listen_address, port)
+        configure_grafana_datasource(host, prometheus_url) if prometheus_url
+
+        puts "  Grafana available at: http://#{host}:#{port}"
+      end
+
+      def configure_grafana_service(host, listen_address, port)
+        env_lines = []
+        env_lines << "Environment=\"GF_SERVER_HTTP_ADDR=#{listen_address}\"" unless listen_address.empty?
+        env_lines << "Environment=\"GF_SERVER_HTTP_PORT=#{port}\"" if port && port != 3000
+        return if env_lines.empty?
+
+        override = <<~CONF
+          [Service]
+          #{env_lines.join("\n  ")}
+        CONF
+
+        ssh_executor.execute_on_host(host) do
+          execute :sudo, 'mkdir', '-p', '/etc/systemd/system/grafana-server.service.d'
+          upload! StringIO.new(override), '/tmp/active_postgres_grafana.override'
+          execute :sudo, 'mv', '/tmp/active_postgres_grafana.override',
+                  '/etc/systemd/system/grafana-server.service.d/override.conf'
+          execute :sudo, 'chown', 'root:root', '/etc/systemd/system/grafana-server.service.d/override.conf'
+          execute :sudo, 'chmod', '644', '/etc/systemd/system/grafana-server.service.d/override.conf'
+          execute :sudo, 'systemctl', 'daemon-reload'
+          execute :sudo, 'systemctl', 'restart', 'grafana-server'
+        end
+      end
+
+      def uninstall_grafana_if_enabled
+        monitoring_config = config.component_config(:monitoring)
+        grafana_config = monitoring_config[:grafana] || {}
+        return unless grafana_config[:enabled]
+
+        host = grafana_config[:host].to_s.strip
+        return if host.empty?
+
+        ssh_executor.execute_on_host(host) do
+          execute :sudo, 'systemctl', 'stop', 'grafana-server'
+          execute :sudo, 'systemctl', 'disable', 'grafana-server'
+          execute :sudo, 'apt-get', 'remove', '-y', 'grafana'
+          execute :sudo, 'rm', '-f', '/etc/apt/sources.list.d/grafana.list'
+          execute :sudo, 'rm', '-f', '/usr/share/keyrings/grafana.gpg'
+          execute :sudo, 'rm', '-rf', '/etc/grafana/provisioning/datasources/active_postgres.yml'
+        end
+      end
+
+      def configure_grafana_datasource(host, prometheus_url)
+        datasource = <<~YAML
+          apiVersion: 1
+          datasources:
+            - name: Prometheus
+              type: prometheus
+              access: proxy
+              url: #{prometheus_url}
+              isDefault: true
+        YAML
+
+        ssh_executor.execute_on_host(host) do
+          execute :sudo, 'mkdir', '-p', '/etc/grafana/provisioning/datasources'
+          upload! StringIO.new(datasource), '/tmp/active_postgres_grafana_ds.yml'
+          execute :sudo, 'mv', '/tmp/active_postgres_grafana_ds.yml', '/etc/grafana/provisioning/datasources/active_postgres.yml'
+          execute :sudo, 'chown', 'root:root', '/etc/grafana/provisioning/datasources/active_postgres.yml'
+          execute :sudo, 'chmod', '644', '/etc/grafana/provisioning/datasources/active_postgres.yml'
+          execute :sudo, 'systemctl', 'restart', 'grafana-server'
+        end
+      end
+
+      def normalize_grafana_password(raw_password)
+        password = raw_password.to_s.rstrip
+
+        raise Error, 'grafana_admin_password secret is missing (required for grafana)' if password.empty?
+
+        password
+      end
+
       def escape_systemd_env(value)
         value.to_s.gsub('\\', '\\\\').gsub('"', '\\"')
       end

data/lib/active_postgres/components/pgbackrest.rb

@@ -20,6 +20,7 @@ module ActivePostgres
           ssh_executor.execute_on_host(host) do
             execute :sudo, 'apt-get', 'remove', '-y', 'pgbackrest'
             execute :sudo, 'rm', '-f', '/etc/cron.d/pgbackrest-backup'
+            execute :sudo, 'rm', '-f', '/etc/cron.d/pgbackrest-backup-incremental'
           end
         end
       end
@@ -37,9 +38,12 @@ module ActivePostgres
         puts "Running #{type} backup..."
         postgres_user = config.postgres_user
 
+        output = nil
         ssh_executor.execute_on_host(config.primary_host) do
-          execute :sudo, '-u', postgres_user, 'pgbackrest', '--stanza=main', "--type=#{type}", 'backup'
+          output = capture(:sudo, '-u', postgres_user, 'pgbackrest', '--stanza=main', "--type=#{type}", 'backup')
         end
+
+        puts output if output
       end
 
       def run_restore(backup_id)
@@ -51,20 +55,43 @@ module ActivePostgres
           execute :sudo, 'systemctl', 'stop', 'postgresql'
 
           # Restore
-          execute :sudo, '-u', postgres_user, 'pgbackrest', '--stanza=main', "--set=#{backup_id}", 'restore'
+          output = capture(:sudo, '-u', postgres_user, 'pgbackrest', '--stanza=main', "--set=#{backup_id}", 'restore')
+          puts output if output
 
           # Start PostgreSQL
           execute :sudo, 'systemctl', 'start', 'postgresql'
         end
       end
 
+      def run_restore_at(target_time, target_action: 'promote')
+        puts "Restoring to #{target_time} (PITR)..."
+        postgres_user = config.postgres_user
+
+        ssh_executor.execute_on_host(config.primary_host) do
+          execute :sudo, 'systemctl', 'stop', 'postgresql'
+
+          output = capture(:sudo, '-u', postgres_user, 'pgbackrest',
+                           '--stanza=main',
+                           '--type=time',
+                           "--target=#{target_time}",
+                           "--target-action=#{target_action}",
+                           'restore')
+          puts output if output
+
+          execute :sudo, 'systemctl', 'start', 'postgresql'
+        end
+      end
+
       def list_backups
         puts 'Available backups:'
         postgres_user = config.postgres_user
 
+        output = nil
         ssh_executor.execute_on_host(config.primary_host) do
-          execute :sudo, '-u', postgres_user, 'pgbackrest', 'info'
+          output = capture(:sudo, '-u', postgres_user, 'pgbackrest', 'info')
         end
+
+        puts output if output
       end
 
       private
@@ -108,13 +135,35 @@ module ActivePostgres
         end
 
         # Set up scheduled backups on primary only
-        if create_stanza && pgbackrest_config[:schedule]
-          setup_backup_schedule(host, pgbackrest_config[:schedule])
+        if create_stanza
+          setup_backup_schedules(host, pgbackrest_config)
+        end
+      end
+
+      def setup_backup_schedules(host, pgbackrest_config)
+        schedules = backup_schedules(pgbackrest_config)
+        schedules.each do |entry|
+          setup_backup_schedule(host, entry[:schedule], entry[:type], entry[:file])
+        end
+      end
+
+      def backup_schedules(pgbackrest_config)
+        schedule_full = pgbackrest_config[:schedule_full] || pgbackrest_config[:schedule]
+        schedule_incremental = pgbackrest_config[:schedule_incremental]
+
+        schedules = []
+        if schedule_full
+          schedules << { type: 'full', schedule: schedule_full, file: '/etc/cron.d/pgbackrest-backup' }
         end
+        if schedule_incremental
+          schedules << { type: 'incremental', schedule: schedule_incremental, file: '/etc/cron.d/pgbackrest-backup-incremental' }
+        end
+
+        schedules
       end
 
-      def setup_backup_schedule(host, schedule)
-        puts "  Setting up backup schedule: #{schedule}"
+      def setup_backup_schedule(host, schedule, backup_type = 'full', cron_file = '/etc/cron.d/pgbackrest-backup')
+        puts "  Setting up #{backup_type} backup schedule: #{schedule}"
         postgres_user = config.postgres_user
 
         # Create cron job for scheduled backups
@@ -123,17 +172,18 @@ module ActivePostgres
           # pgBackRest scheduled backups (managed by active_postgres)
           SHELL=/bin/bash
           PATH=/usr/local/sbin:/usr/local/bin:/sbin:/bin:/usr/sbin:/usr/bin
-          #{schedule} #{postgres_user} pgbackrest --stanza=main --type=full backup
+          #{schedule} #{postgres_user} pgbackrest --stanza=main --type=#{backup_type} backup
         CRON
 
         # Install cron job in /etc/cron.d (system cron directory)
         # Use a temp file + sudo mv to avoid shell redirection permission issues.
-        ssh_executor.upload_file(host, cron_content, '/etc/cron.d/pgbackrest-backup', mode: '644', owner: 'root:root')
+        ssh_executor.upload_file(host, cron_content, cron_file, mode: '644', owner: 'root:root')
       end
 
       def remove_backup_schedule(host)
         ssh_executor.execute_on_host(host) do
           execute :sudo, 'rm', '-f', '/etc/cron.d/pgbackrest-backup'
+          execute :sudo, 'rm', '-f', '/etc/cron.d/pgbackrest-backup-incremental'
         end
       end
     end

data/lib/active_postgres/components/pgbouncer.rb

@@ -5,7 +5,8 @@ module ActivePostgres
         puts 'Installing PgBouncer for connection pooling...'
 
         config.all_hosts.each do |host|
-          install_on_host(host)
+          is_standby = config.standby_hosts.include?(host)
+          install_on_host(host, is_standby: is_standby)
         end
       end
 
@@ -49,16 +50,26 @@ module ActivePostgres
 
       def install_on_standby(standby_host)
         puts "Installing PgBouncer on standby #{standby_host}..."
-        install_on_host(standby_host)
+        install_on_host(standby_host, is_standby: true)
       end
 
       private
 
-      def install_on_host(host)
+      def install_on_host(host, is_standby: false)
         puts "  Installing PgBouncer on #{host}..."
 
         user_config = config.component_config(:pgbouncer)
-        follow_primary = user_config[:follow_primary] == true
+
+        # Determine follow_primary behavior:
+        # - Primary: use global follow_primary setting
+        # - Standby: check per-standby pgbouncer_follow_primary, default false (use localhost for read replicas)
+        if is_standby
+          standby_config = config.standby_config_for(host) || {}
+          follow_primary = standby_config['pgbouncer_follow_primary'] == true
+        else
+          follow_primary = user_config[:follow_primary] == true
+        end
+
         if follow_primary && !config.component_enabled?(:repmgr)
           raise Error, 'PgBouncer follow_primary requires repmgr to be enabled'
         end
@@ -67,7 +78,8 @@ module ActivePostgres
         optimal_pool = ConnectionPooler.calculate_optimal_pool_sizes(max_connections)
 
         pgbouncer_config = optimal_pool.merge(user_config)
-        pgbouncer_config[:database_host] = config.primary_replication_host if follow_primary
+        # For standbys not following primary, use localhost; otherwise use primary host
+        pgbouncer_config[:database_host] = follow_primary ? config.primary_replication_host : '127.0.0.1'
         ssl_enabled = config.component_enabled?(:ssl)
         has_ca_cert = ssl_enabled && secrets.resolve('ssl_chain')
         secrets_obj = secrets