manageiq-postgres_ha_admin 3.1.4 → 3.2.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 302246b3dedf536233f20df1c381932adbbe7c85366e400dd3678eda57f8ebf4
-  data.tar.gz: cfc4aee15059200a0dddd56dea7a6f4563b89b24742682dd1c10b66e26bd9ac5
+  metadata.gz: 7c6e01b8391b55a231ec81a0e050a97b2ccd02cc1fb576dc9272021f2130e593
+  data.tar.gz: d0b27f3b48ab6e843175d740e98bc6758d3facb9815c97dfa1de3147072f2fe2
 SHA512:
-  metadata.gz: 3169f9907e9b75c426dfea5908945921e82ef27a68769427ae556ab703d683ca0a5b4887d7206526417acdee0f908a1660ddbc90717d18ff09fabd873b8b5b6b
-  data.tar.gz: 15766ce8377afa98450b7928d46dc790fb2a9f874760914fa929583d4951bae18288c1ff3b5dff7e75d07247723c17e0db87d71703fb22f7103db0d7e573b875
+  metadata.gz: dad10a7c85d3d5a30ced79733e407a84584fc3cd32ff6c0c77350cf3f84cfa3f43206ad2e806499ba8f28f6804296f8c7c4d45fe6ccb4fc09899ce62ca388540
+  data.tar.gz: 8b80044d67bf6f6215f59f3e21a9c88b9b32fb8d8eb402881e582a7c869eb25d81409076d4dde8def21778e831b391bd43704384a0ee317c5b9f16caa549f3af

data/.github/workflows/ci.yaml

@@ -40,6 +40,7 @@ jobs:
       with:
         ruby-version: ${{ matrix.ruby-version }}
         bundler-cache: true
+      timeout-minutes: 30
     - name: Run tests
       run: bundle exec rake
     - name: Report code coverage

data/CHANGELOG.md

@@ -5,36 +5,50 @@ This project adheres to [Semantic Versioning](http://semver.org/).
 
 ## [Unreleased]
 
+## [3.2.1] - 2023-03-22
+### Changed
+- Improve failover logging [#39](https://github.com/ManageIQ/manageiq-postgres_ha_admin/pull/39)
+
+## [3.2.0] - 2022-09-26
+### Added
+- Add simple prefix to all log lines [#38](https://github.com/ManageIQ/manageiq-postgres_ha_admin/pull/38)
+
+### Changed
+- 5 minute db checks is too long, use 2 minutes [#37](https://github.com/ManageIQ/manageiq-postgres_ha_admin/pull/37)
+
+### Removed
+- Remove unused pglogical_config_handler [#35](https://github.com/ManageIQ/manageiq-postgres_ha_admin/pull/35)
+
+### Fixed
+- Don't attempt a database failover without a known standby [#36](https://github.com/ManageIQ/manageiq-postgres_ha_admin/pull/36)
+
 ## [3.1.4] - 2022-04-30
 ### Changed
-- Loosed rails dependency to include rails 6.1
-- Support Ruby 3.0
+- Loosen rails dependency to include rails 6.1 [#28](https://github.com/ManageIQ/manageiq-postgres_ha_admin/pull/28)
 
 ## [3.1.3] - 2021-09-27
-
 ### Changed
 - Loosen manageiq-password dependency to < 2 [#26](https://github.com/ManageIQ/manageiq-postgres_ha_admin/pull/26)
-- Update ruby versions to 2.5.8 minimum [#25](https://github.com/ManageIQ/manageiq-postgres_ha_admin/pull/25)
+- Update ruby versions to 2.5.8 minimum and support Ruby 3.0 [#25](https://github.com/ManageIQ/manageiq-postgres_ha_admin/pull/25)
 - Switch to manageiq-style [#24](https://github.com/ManageIQ/manageiq-postgres_ha_admin/pull/24)
 
 ## [3.1.2] - 2020-12-21
+### Changed
 - Allow for Rails 6.0 [#23](https://github.com/ManageIQ/manageiq-postgres_ha_admin/pull/23)
 
 ## [3.1.1] - 2019-12-12
+### Changed
 - Remove the dependency on linux_admin [#19](https://github.com/ManageIQ/manageiq-postgres_ha_admin/pull/19)
 
 ## [3.1.0] - 2019-05-09
-
 ### Added
 - Add a logical replication config handler [#17](https://github.com/ManageIQ/manageiq-postgres_ha_admin/pull/17)
 
 ## [3.0.1] - 2019-03-19
-
 ### Changed
 - Remove references to MiqPassword [#15](https://github.com/ManageIQ/manageiq-postgres_ha_admin/pull/15)
 
 ## [3.0.0] - 2018-09-05
-
 ### Added
 - Allow users of the gem to specify a logger object [#7](https://github.com/ManageIQ/manageiq-postgres_ha_admin/pull/7)
 
@@ -44,7 +58,6 @@ This project adheres to [Semantic Versioning](http://semver.org/).
 - Make failover monitor generic [#10](https://github.com/ManageIQ/manageiq-postgres_ha_admin/pull/10)
 
 ## [2.0.0] - 2018-08-01
-
 ### Added
 - Add pg-dsn_parser to the gemspec [#5](https://github.com/ManageIQ/manageiq-postgres_ha_admin/pull/5)
 - Add postgresql addon for travis [#6](https://github.com/ManageIQ/manageiq-postgres_ha_admin/pull/6)
@@ -52,7 +65,9 @@ This project adheres to [Semantic Versioning](http://semver.org/).
 ### Changed
 - Make changes for upgrading repmgr to version 4 [#4](https://github.com/ManageIQ/manageiq-postgres_ha_admin/pull/4)
 
-[Unreleased]: https://github.com/ManageIQ/manageiq-postgres_ha_admin/compare/v3.1.4...master
+[Unreleased]: https://github.com/ManageIQ/manageiq-postgres_ha_admin/compare/v3.2.1...master
+[3.2.1]: https://github.com/ManageIQ/manageiq-postgres_ha_admin/compare/v3.2.0...v3.2.1
+[3.2.0]: https://github.com/ManageIQ/manageiq-postgres_ha_admin/compare/v3.1.4...v3.2.0
 [3.1.4]: https://github.com/ManageIQ/manageiq-postgres_ha_admin/compare/v3.1.3...v3.1.4
 [3.1.3]: https://github.com/ManageIQ/manageiq-postgres_ha_admin/compare/v3.1.2...v3.1.3
 [3.1.2]: https://github.com/ManageIQ/manageiq-postgres_ha_admin/compare/v3.1.1...v3.1.2

data/lib/manageiq/postgres_ha_admin/config_handler.rb

@@ -27,9 +27,12 @@ module PostgresHaAdmin
       @before_failover_cb&.call
     end
 
+    # Upon successful failover
     def do_after_failover(new_primary_conn_info)
       @after_failover_cb&.call(new_primary_conn_info)
     end
+
+    # If needed, we can add an unsuccessful failover hook
   end
 end
 end

data/lib/manageiq/postgres_ha_admin/failover_monitor.rb

@@ -7,7 +7,7 @@ module PostgresHaAdmin
     include Logging
 
     FAILOVER_ATTEMPTS = 10
-    DB_CHECK_FREQUENCY = 300
+    DB_CHECK_FREQUENCY = 120
     FAILOVER_CHECK_FREQUENCY = 60
     attr_accessor :failover_attempts, :db_check_frequency, :failover_check_frequency
     attr_reader :config_handlers
@@ -26,22 +26,22 @@ module PostgresHaAdmin
         begin
           connection = pg_connection(handler.read)
           if connection
-            server_store.update_servers(connection)
+            server_store.update_servers(connection, handler.name)
             connection.finish
             next
           end
 
-          logger.error("Primary Database is not available for #{handler.name}. Starting to execute failover...")
+          log_settings
+          server_store.log_current_server_store(handler.name)
+          logger.error("#{log_prefix(__callee__)} Primary Database is not available for #{handler.name}. Starting to execute failover...")
           handler.do_before_failover
 
           new_conn_info = execute_failover(handler, server_store)
-          if new_conn_info
-            handler.do_after_failover(new_conn_info)
-          else
-            logger.error("Failover failed")
-          end
+
+          # Upon success, we pass a connection hash
+          handler.do_after_failover(new_conn_info) if new_conn_info
         rescue => e
-          logger.error("Received #{e.class} error while monitoring #{handler.name}: #{e.message}")
+          logger.error("#{log_prefix(__callee__)} Received #{e.class} error while monitoring #{handler.name}: #{e.message}")
           logger.error(e.backtrace)
         end
       end
@@ -52,7 +52,7 @@ module PostgresHaAdmin
         begin
           monitor
         rescue => err
-          logger.error("#{err.class}: #{err}")
+          logger.error("#{log_prefix(__callee__)} #{err.class}: #{err}")
           logger.error(err.backtrace.join("\n"))
         end
         sleep(db_check_frequency)
@@ -67,37 +67,59 @@ module PostgresHaAdmin
 
     private
 
+    def log_settings
+      logger.info("#{log_prefix(__callee__)} Current HA settings: FAILOVER_ATTEMPTS=#{@failover_attempts} DB_CHECK_FREQUENCY=#{@db_check_frequency} FAILOVER_CHECK_FREQUENCY=#{@failover_check_frequency}")
+    end
+
     def initialize_settings(ha_admin_yml_file)
       ha_admin_yml = {}
       begin
         ha_admin_yml = YAML.load_file(ha_admin_yml_file) if File.exist?(ha_admin_yml_file)
       rescue SystemCallError, IOError => err
-        logger.error("#{err.class}: #{err}")
-        logger.info("File not loaded: #{ha_admin_yml_file}. Default settings for failover will be used.")
+        logger.error("#{log_prefix(__callee__)} #{err.class}: #{err}")
+        logger.info("#{log_prefix(__callee__)} File not loaded: #{ha_admin_yml_file}. Default settings for failover will be used.")
       end
       @failover_attempts = ha_admin_yml['failover_attempts'] || FAILOVER_ATTEMPTS
       @db_check_frequency = ha_admin_yml['db_check_frequency'] || DB_CHECK_FREQUENCY
       @failover_check_frequency = ha_admin_yml['failover_check_frequency'] || FAILOVER_CHECK_FREQUENCY
-      logger.info("FAILOVER_ATTEMPTS=#{@failover_attempts} DB_CHECK_FREQUENCY=#{@db_check_frequency} FAILOVER_CHECK_FREQUENCY=#{@failover_check_frequency}")
+      log_settings
+    end
+
+    def any_known_standby?(handler, server_store)
+      current_host = handler.read[:host]
+      server_store.servers.any? do |server|
+        server[:host] != current_host && server[:type] == "standby"
+      end
     end
 
     def execute_failover(handler, server_store)
+      # TODO: Instead of returning false, we should raise:
+      # "No active standby"
+      # "Standby in recovery"
+      # "Exhausted all failover retry attempts" exceptions
+      unless any_known_standby?(handler, server_store)
+        logger.error("#{log_prefix(__callee__)} Cannot attempt failover without a known active standby for #{handler.name}.  Please verify the database.yml and ensure the database is started.")
+        return false
+      end
+
       failover_attempts.times do
         with_each_standby_connection(handler, server_store) do |connection, params|
           next if database_in_recovery?(connection)
           next unless server_store.host_is_primary?(params[:host], connection)
-          logger.info("Failing over to server using conninfo: #{params.reject { |k, _v| k == :password }}")
-          server_store.update_servers(connection)
+          logger.info("#{log_prefix(__callee__)} Failing over for #{handler.name} to server using conninfo: #{server_store.sanitized_connection_parameters(params)}")
+          server_store.update_servers(connection, handler.name)
           handler.write(params)
           return params
         end
         sleep(failover_check_frequency)
       end
+      logger.error("#{log_prefix(__callee__)} Failover failed for #{handler.name}")
       false
     end
 
     def with_each_standby_connection(handler, server_store)
       active_servers_conninfo(handler, server_store).each do |params|
+        logger.info("#{log_prefix(__callee__)} Checking active server for #{handler.name} using conninfo: #{server_store.sanitized_connection_parameters(params)}")
         connection = pg_connection(params)
         next if connection.nil?
         begin
@@ -111,7 +133,7 @@ module PostgresHaAdmin
     def pg_connection(params)
       PG::Connection.open(params)
     rescue PG::Error => e
-      logger.error("Failed to establish PG connection: #{e.message}")
+      logger.error("#{log_prefix(__callee__)} Failed to establish PG connection: #{e.message}")
       nil
     end
 

data/lib/manageiq/postgres_ha_admin/logging.rb

@@ -4,6 +4,10 @@ module PostgresHaAdmin
     def logger
       ManageIQ::PostgresHaAdmin.logger
     end
+
+    def log_prefix(method)
+      "(PostgresHaAdmin##{method})"
+    end
   end
 end
 end

data/lib/manageiq/postgres_ha_admin/server_store.rb

@@ -22,14 +22,15 @@ module PostgresHaAdmin
       end
     end
 
-    def update_servers(connection)
+    def update_servers(connection, handler_name = "unspecified handler")
       new_servers = query_repmgr(connection)
       if servers_changed?(new_servers)
-        logger.info("Updating servers cache to #{new_servers}")
+        log_current_server_store(handler_name)
+        logger.info("#{log_prefix(__callee__)} Updating servers cache to #{sanitized_servers_array(new_servers)} for #{handler_name}")
         @servers = new_servers
       end
     rescue IOError => err
-      logger.error("#{err.class}: #{err}")
+      logger.error("#{log_prefix(__callee__)} #{err.class}: #{err}")
       logger.error(err.backtrace.join("\n"))
     end
 
@@ -42,6 +43,18 @@ module PostgresHaAdmin
       false
     end
 
+    def log_current_server_store(handler_name)
+      logger.info("#{log_prefix(__callee__)} Current servers cache: #{sanitized_servers_array(@servers)} for #{handler_name}")
+    end
+
+    def sanitized_servers_array(server_array)
+      server_array.collect { |p| sanitized_connection_parameters(p) }
+    end
+
+    def sanitized_connection_parameters(params)
+      params.reject { |k, _v| k.to_s == "password" }
+    end
+
     private
 
     def servers_changed?(new_servers)
@@ -59,7 +72,7 @@ module PostgresHaAdmin
       db_result.clear
       result
     rescue PG::Error => err
-      logger.error("#{err.class}: #{err}")
+      logger.error("#{log_prefix(__callee__)} #{err.class}: #{err}")
       logger.error(err.backtrace.join("\n"))
       result
     end

data/lib/manageiq/postgres_ha_admin/version.rb

@@ -1,5 +1,5 @@
 module ManageIQ
   module PostgresHaAdmin
-    VERSION = '3.1.4'.freeze
+    VERSION = '3.2.1'.freeze
   end
 end

data/lib/manageiq-postgres_ha_admin.rb

@@ -8,7 +8,6 @@ require 'manageiq/postgres_ha_admin/failover_monitor'
 
 require 'manageiq/postgres_ha_admin/config_handler'
 require 'manageiq/postgres_ha_admin/config_handler/rails_config_handler'
-require 'manageiq/postgres_ha_admin/config_handler/pglogical_config_handler'
 require 'manageiq/postgres_ha_admin/config_handler/logical_replication_config_handler'
 
 module ManageIQ

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: manageiq-postgres_ha_admin
 version: !ruby/object:Gem::Version
-  version: 3.1.4
+  version: 3.2.1
 platform: ruby
 authors:
 - ManageIQ Developers
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2022-04-29 00:00:00.000000000 Z
+date: 2023-03-22 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activesupport
@@ -180,7 +180,6 @@ files:
 - lib/manageiq-postgres_ha_admin.rb
 - lib/manageiq/postgres_ha_admin/config_handler.rb
 - lib/manageiq/postgres_ha_admin/config_handler/logical_replication_config_handler.rb
-- lib/manageiq/postgres_ha_admin/config_handler/pglogical_config_handler.rb
 - lib/manageiq/postgres_ha_admin/config_handler/rails_config_handler.rb
 - lib/manageiq/postgres_ha_admin/failover_monitor.rb
 - lib/manageiq/postgres_ha_admin/logging.rb
@@ -210,7 +209,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.3.5
+rubygems_version: 3.2.33
 signing_key:
 specification_version: 4
 summary: ManageIQ Postgres H.A. Admin

data/lib/manageiq/postgres_ha_admin/config_handler/pglogical_config_handler.rb

@@ -1,36 +0,0 @@
-require 'pg'
-require 'pg/dsn_parser'
-
-module ManageIQ
-module PostgresHaAdmin
-  class PglogicalConfigHandler < ConfigHandler
-    attr_reader :subscription, :conn_info
-
-    def initialize(options = {})
-      @subscription = options[:subscription]
-      @conn_info    = options[:conn_info]
-    end
-
-    def name
-      "pglogical subscription #{subscription} Config Handler"
-    end
-
-    def read
-      conn = PG::Connection.open(@conn_info)
-      dsn = conn.exec_params(<<~SQL, [@subscription]).first["if_dsn"]
-        SELECT if_dsn
-        FROM pglogical.subscription s
-        JOIN pglogical.node_interface i
-          ON s.sub_origin_if = i.if_id
-        WHERE s.sub_name = $1
-      SQL
-      PG::DSNParser.new.parse(dsn)
-    end
-
-    def write(_params)
-      # Nothing to do here as the expectation is that the user will 
-      # remove and re-add the subscription in the after failover callback
-    end
-  end
-end
-end