manageiq-postgres_ha_admin 2.0.0 → 3.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 2ea7e89193add948ecd827491eae6c7e055c20a6
-  data.tar.gz: d416a3b5fd696ca582e13e360b6ea00af762aa58
+  metadata.gz: 00a1ceacb4bfb538a4cdeb7b7b1bc9d60e7ca986
+  data.tar.gz: 7ace66668500095929cb06389dd98ed77ec9c2ce
 SHA512:
-  metadata.gz: a76ffe81cb4c0f218a5374a333f2458da83c59ff14cbc285ba9787c58e4362568c7c0303d794f587098ae3aa4b6fe144fc44c6ef1270e990e1581e5c83521394
-  data.tar.gz: 2aaeb1824891fad95440f473ccd864a048538ddcdf37832a9d3b8c492d29d81e1a1d353ee070aca4861136a5cea8a2152bd26bf90c94c989df29e2f1bd97e088
+  metadata.gz: ee0f5eb9fb4d357e9d413166c9251cb67191bc7110d2b275eb6d22055a923a1e85f9313d023f13ff156690a82d76aa05fca79dc60d257b79c246db80d0338fa8
+  data.tar.gz: 6444e2e66980ee5c3f498ee292e47f9cf2a88f4169106690fa41a0706f14b1a8e802a2020d06180577920025cd7212dc3801e2a37bb577f0eee35ab165352a85

data/CHANGELOG.md

@@ -5,6 +5,16 @@ This project adheres to [Semantic Versioning](http://semver.org/).
 
 ## [Unreleased]
 
+## [3.0.0] - 2018-09-05
+
+### Added
+- Allow users of the gem to specify a logger object [#7](https://github.com/ManageIQ/manageiq-postgres_ha_admin/pull/7)
+
+### Changed
+- Make sources of database connection info pluggable [#8](https://github.com/ManageIQ/manageiq-postgres_ha_admin/pull/8)
+- Improve FailoverDatabases/ServerStore class [#9](https://github.com/ManageIQ/manageiq-postgres_ha_admin/pull/9)
+- Make failover monitor generic [#10](https://github.com/ManageIQ/manageiq-postgres_ha_admin/pull/10)
+
 ## [2.0.0] - 2018-08-01
 
 ### Added
@@ -14,5 +24,6 @@ This project adheres to [Semantic Versioning](http://semver.org/).
 ### Changed
 - Make changes for upgrading repmgr to version 4 [#4](https://github.com/ManageIQ/manageiq-postgres_ha_admin/pull/4)
 
-[Unreleased]: https://github.com/ManageIQ/manageiq-postgres_ha_admin/compare/v2.0.0...master
+[Unreleased]: https://github.com/ManageIQ/manageiq-postgres_ha_admin/compare/v3.0.0...master
+[3.0.0]: https://github.com/ManageIQ/manageiq-postgres_ha_admin/compare/v2.0.0...v3.0.0
 [2.0.0]: https://github.com/ManageIQ/manageiq-postgres_ha_admin/compare/v1.0.0...v2.0.0

data/lib/manageiq-postgres_ha_admin.rb

@@ -1,4 +1,23 @@
 require 'manageiq/postgres_ha_admin/version'
-require 'manageiq/postgres_ha_admin/database_yml'
-require 'manageiq/postgres_ha_admin/failover_databases'
+
+require 'manageiq/postgres_ha_admin/logging'
+require 'manageiq/postgres_ha_admin/null_logger'
+
+require 'manageiq/postgres_ha_admin/server_store'
 require 'manageiq/postgres_ha_admin/failover_monitor'
+
+require 'manageiq/postgres_ha_admin/config_handler'
+require 'manageiq/postgres_ha_admin/config_handler/rails_config_handler'
+require 'manageiq/postgres_ha_admin/config_handler/pglogical_config_handler'
+
+module ManageIQ
+  module PostgresHaAdmin
+    class << self
+      attr_writer :logger
+    end
+
+    def self.logger
+      @logger ||= NullLogger.new
+    end
+  end
+end

data/lib/manageiq/postgres_ha_admin/config_handler.rb

@@ -0,0 +1,35 @@
+module ManageIQ
+module PostgresHaAdmin
+  class ConfigHandler
+    def name
+      "Config Handler"
+    end
+
+    def read
+      raise NotImplementedError
+    end
+
+    def write(_conninfo)
+      raise NotImplementedError
+    end
+
+    def before_failover(&block)
+      raise ArgumentError, "A block is required to set the before failover callback" unless block_given?
+      @before_failover_cb = block
+    end
+
+    def after_failover(&block)
+      raise ArgumentError, "A block is required to set the after failover callback" unless block_given?
+      @after_failover_cb = block
+    end
+
+    def do_before_failover
+      @before_failover_cb&.call
+    end
+
+    def do_after_failover(new_primary_conn_info)
+      @after_failover_cb&.call(new_primary_conn_info)
+    end
+  end
+end
+end

data/lib/manageiq/postgres_ha_admin/config_handler/pglogical_config_handler.rb

@@ -0,0 +1,36 @@
+require 'pg'
+require 'pg/dsn_parser'
+
+module ManageIQ
+module PostgresHaAdmin
+  class PglogicalConfigHandler < ConfigHandler
+    attr_reader :subscription, :conn_info
+
+    def initialize(options = {})
+      @subscription = options[:subscription]
+      @conn_info    = options[:conn_info]
+    end
+
+    def name
+      "pglogical subscription #{subscription} Config Handler"
+    end
+
+    def read
+      conn = PG::Connection.open(@conn_info)
+      dsn = conn.exec_params(<<~SQL, [@subscription]).first["if_dsn"]
+        SELECT if_dsn
+        FROM pglogical.subscription s
+        JOIN pglogical.node_interface i
+          ON s.sub_origin_if = i.if_id
+        WHERE s.sub_name = $1
+      SQL
+      PG::DSNParser.new.parse(dsn)
+    end
+
+    def write(_params)
+      # Nothing to do here as the expectation is that the user will 
+      # remove and re-add the subscription in the after failover callback
+    end
+  end
+end
+end

data/lib/manageiq/postgres_ha_admin/{database_yml.rb → config_handler/rails_config_handler.rb}

@@ -4,29 +4,33 @@ require 'fileutils'
 
 module ManageIQ
 module PostgresHaAdmin
-  class DatabaseYml
-    attr_reader :db_yml_file, :environment
+  class RailsConfigHandler < ConfigHandler
+    attr_reader :environment, :file_path
 
-    def initialize(db_yml_file, environment)
-      @db_yml_file = db_yml_file
-      @environment = environment
+    def initialize(options = {})
+      @file_path   = options[:file_path]
+      @environment = options[:environment]
     end
 
-    def pg_params_from_database_yml
-      rails_params_to_pg(YAML.load_file(db_yml_file)[environment])
+    def name
+      "Rails #{environment} Config Handler"
     end
 
-    def update_database_yml(params)
-      db_yml = YAML.load_file(db_yml_file)
+    def read
+      rails_params_to_pg(YAML.load_file(file_path)[environment])
+    end
+
+    def write(params)
+      db_yml = YAML.load_file(file_path)
       db_yml[environment].merge!(pg_parameters_to_rails(params))
       remove_empty(db_yml[environment])
 
-      new_name = "#{db_yml_file}_#{Time.current.strftime("%d-%B-%Y_%H.%M.%S")}"
-      FileUtils.copy(db_yml_file, new_name)
+      new_name = "#{file_path}_#{Time.current.strftime("%d-%B-%Y_%H.%M.%S")}"
+      FileUtils.copy(file_path, new_name)
       begin
-        File.write(db_yml_file, db_yml.to_yaml)
+        File.write(file_path, db_yml.to_yaml)
       rescue
-        FileUtils.mv(new_name, db_yml_file)
+        FileUtils.mv(new_name, file_path)
         raise
       end
       new_name

data/lib/manageiq/postgres_ha_admin/failover_monitor.rb

@@ -5,48 +5,46 @@ require 'linux_admin'
 module ManageIQ
 module PostgresHaAdmin
   class FailoverMonitor
-    RAILS_ROOT = [
-      Pathname.new("/var/www/miq/vmdb"),
-      Pathname.new(File.expand_path(File.join(__dir__, "../..")))
-    ].detect { |f| File.exist?(f) }
+    include Logging
+
     FAILOVER_ATTEMPTS = 10
     DB_CHECK_FREQUENCY = 300
     FAILOVER_CHECK_FREQUENCY = 60
-    LOG_FILE = '/var/www/miq/vmdb/log/ha_admin.log'.freeze
     attr_accessor :failover_attempts, :db_check_frequency, :failover_check_frequency
+    attr_reader :config_handlers
 
-    def initialize(db_yml_file: '/var/www/miq/vmdb/config/database.yml',
-                   failover_yml_file: '/var/www/miq/vmdb/config/failover_databases.yml',
-                   ha_admin_yml_file: '/var/www/miq/vmdb/config/ha_admin.yml',
-                   logger: nil,
-                   environment: 'production')
-      if logger.respond_to?(:error, :info)
-        @logger = logger
-      else
-        @logger = Logger.new(LOG_FILE)
-        @logger.level = Logger::INFO
-      end
-      @database_yml = DatabaseYml.new(db_yml_file, environment)
-      @failover_db = FailoverDatabases.new(failover_yml_file, @logger)
-      initialize_settings(ha_admin_yml_file)
+    def initialize(config_path = "")
+      initialize_settings(config_path)
+      @config_handlers = []
     end
 
-    def monitor
-      connection = pg_connection(@database_yml.pg_params_from_database_yml)
-      if connection
-        @failover_db.update_failover_yml(connection)
-        connection.finish
-        return
-      end
-
-      @logger.error("Primary Database is not available. EVM server stop initiated. Starting to execute failover...")
-      stop_evmserverd
+    def add_handler(handler)
+      @config_handlers << [handler, ServerStore.new]
+    end
 
-      if execute_failover
-        start_evmserverd
-        raise_failover_event
-      else
-        @logger.error("Failover failed")
+    def monitor
+      config_handlers.each do |handler, server_store|
+        begin
+          connection = pg_connection(handler.read)
+          if connection
+            server_store.update_servers(connection)
+            connection.finish
+            next
+          end
+
+          logger.error("Primary Database is not available for #{handler.name}. Starting to execute failover...")
+          handler.do_before_failover
+
+          new_conn_info = execute_failover(handler, server_store)
+          if new_conn_info
+            handler.do_after_failover(new_conn_info)
+          else
+            logger.error("Failover failed")
+          end
+        rescue => e
+          logger.error("Received #{e.class} error while monitoring #{handler.name}: #{e.message}")
+          logger.error(e.backtrace)
+        end
       end
     end
 
@@ -55,59 +53,52 @@ module PostgresHaAdmin
         begin
           monitor
         rescue => err
-          @logger.error("#{err.class}: #{err}")
-          @logger.error(err.backtrace.join("\n"))
+          logger.error("#{err.class}: #{err}")
+          logger.error(err.backtrace.join("\n"))
         end
         sleep(db_check_frequency)
       end
     end
 
-    def active_servers_conninfo
-      servers = @failover_db.active_databases_conninfo_hash
-      db_yml_params = @database_yml.pg_params_from_database_yml
-      servers.map! { |info| db_yml_params.merge(info) }
-    end
-
-    def raise_failover_event
-      require "awesome_spawn"
-      AwesomeSpawn.run("rake evm:raise_server_event",
-                       :chdir  => RAILS_ROOT,
-                       :params => ["--", {:event  => "db_failover_executed"}])
+    def active_servers_conninfo(handler, server_store)
+      servers = server_store.connection_info_list
+      current_params = handler.read
+      servers.map! { |info| current_params.merge(info) }
     end
 
     private
 
     def initialize_settings(ha_admin_yml_file)
+      ha_admin_yml = {}
       begin
-        ha_admin_yml = YAML.load_file(ha_admin_yml_file)
+        ha_admin_yml = YAML.load_file(ha_admin_yml_file) if File.exist?(ha_admin_yml_file)
       rescue SystemCallError, IOError => err
-        ha_admin_yml = {}
-        @logger.error("#{err.class}: #{err}")
-        @logger.info("File not loaded: #{ha_admin_yml_file}. Default settings for failover will be used.")
+        logger.error("#{err.class}: #{err}")
+        logger.info("File not loaded: #{ha_admin_yml_file}. Default settings for failover will be used.")
       end
       @failover_attempts = ha_admin_yml['failover_attempts'] || FAILOVER_ATTEMPTS
       @db_check_frequency = ha_admin_yml['db_check_frequency'] || DB_CHECK_FREQUENCY
       @failover_check_frequency = ha_admin_yml['failover_check_frequency'] || FAILOVER_CHECK_FREQUENCY
-      @logger.info("FAILOVER_ATTEMPTS=#{@failover_attempts} DB_CHECK_FREQUENCY=#{@db_check_frequency} FAILOVER_CHECK_FREQUENCY=#{@failover_check_frequency}")
+      logger.info("FAILOVER_ATTEMPTS=#{@failover_attempts} DB_CHECK_FREQUENCY=#{@db_check_frequency} FAILOVER_CHECK_FREQUENCY=#{@failover_check_frequency}")
     end
 
-    def execute_failover
+    def execute_failover(handler, server_store)
       failover_attempts.times do
-        with_each_standby_connection do |connection, params|
+        with_each_standby_connection(handler, server_store) do |connection, params|
           next if database_in_recovery?(connection)
-          next unless @failover_db.host_is_repmgr_primary?(params[:host], connection)
-          @logger.info("Failing over to server using conninfo: #{params.reject { |k, _v| k == :password }}")
-          @failover_db.update_failover_yml(connection)
-          @database_yml.update_database_yml(params)
-          return true
+          next unless server_store.host_is_primary?(params[:host], connection)
+          logger.info("Failing over to server using conninfo: #{params.reject { |k, _v| k == :password }}")
+          server_store.update_servers(connection)
+          handler.write(params)
+          return params
         end
         sleep(failover_check_frequency)
       end
       false
     end
 
-    def with_each_standby_connection
-      active_servers_conninfo.each do |params|
+    def with_each_standby_connection(handler, server_store)
+      active_servers_conninfo(handler, server_store).each do |params|
         connection = pg_connection(params)
         next if connection.nil?
         begin
@@ -121,19 +112,10 @@ module PostgresHaAdmin
     def pg_connection(params)
       PG::Connection.open(params)
     rescue PG::Error => e
-      @logger.error("Failed to establish PG connection: #{e.message}")
+      logger.error("Failed to establish PG connection: #{e.message}")
       nil
     end
 
-    def start_evmserverd
-      LinuxAdmin::Service.new("evmserverd").restart
-      @logger.info("Starting EVM server from failover monitor")
-    end
-
-    def stop_evmserverd
-      LinuxAdmin::Service.new("evmserverd").stop
-    end
-
     # Checks if postgres database is in recovery mode
     #
     # @param pg_connection [PG::Connection] established pg connection

data/lib/manageiq/postgres_ha_admin/logging.rb

@@ -0,0 +1,9 @@
+module ManageIQ
+module PostgresHaAdmin
+  module Logging
+    def logger
+      ManageIQ::PostgresHaAdmin.logger
+    end
+  end
+end
+end

data/lib/manageiq/postgres_ha_admin/null_logger.rb

@@ -0,0 +1,13 @@
+require 'logger'
+
+module ManageIQ
+module PostgresHaAdmin
+  class NullLogger < Logger
+    def initialize(*_args)
+    end
+
+    def add(*_args, &_block)
+    end
+  end
+end
+end

data/lib/manageiq/postgres_ha_admin/{failover_databases.rb → server_store.rb}

@@ -4,38 +4,38 @@ require 'pg/dsn_parser'
 
 module ManageIQ
 module PostgresHaAdmin
-  class FailoverDatabases
+  class ServerStore
+    include Logging
+
     TABLE_NAME = "repmgr.nodes".freeze
 
-    attr_reader :yml_file
+    attr_reader :servers
 
-    def initialize(yml_file, logger)
-      @yml_file = yml_file
-      @logger = logger
+    def initialize
+      @servers = []
     end
 
-    def active_databases_conninfo_hash
+    def connection_info_list
       valid_keys = PG::Connection.conndefaults_hash.keys + [:requiressl]
-      active_databases.map! do |db_info|
+      servers.map! do |db_info|
         db_info.keep_if { |k, _v| valid_keys.include?(k) }
       end
     end
 
-    def active_databases
-      all_databases.select { |record| record[:active] }
-    end
-
-    def update_failover_yml(connection)
-      arr_yml = query_repmgr(connection)
-      File.write(yml_file, arr_yml.to_yaml) unless arr_yml.empty?
+    def update_servers(connection)
+      new_servers = query_repmgr(connection)
+      if servers_changed?(new_servers)
+        logger.info("Updating servers cache to #{new_servers}")
+        @servers = new_servers
+      end
     rescue IOError => err
-      @logger.error("#{err.class}: #{err}")
-      @logger.error(err.backtrace.join("\n"))
+      logger.error("#{err.class}: #{err}")
+      logger.error(err.backtrace.join("\n"))
     end
 
-    def host_is_repmgr_primary?(host, connection)
+    def host_is_primary?(host, connection)
       query_repmgr(connection).each do |record|
-        if record[:host] == host && entry_is_active_master?(record)
+        if record[:host] == host && record[:type] == 'primary'
           return true
         end
       end
@@ -44,10 +44,14 @@ module PostgresHaAdmin
 
     private
 
+    def servers_changed?(new_servers)
+      ((servers - new_servers) + (new_servers - servers)).any?
+    end
+
     def query_repmgr(connection)
       return [] unless table_exists?(connection, TABLE_NAME)
       result = []
-      db_result = connection.exec("SELECT type, conninfo, active FROM #{TABLE_NAME}")
+      db_result = connection.exec("SELECT type, conninfo, active FROM #{TABLE_NAME} WHERE active")
       db_result.map_types!(PG::BasicTypeMapForResults.new(connection)).each do |record|
         dsn = PG::DSNParser.parse(record.delete("conninfo"))
         result << record.symbolize_keys.merge(dsn)
@@ -55,20 +59,11 @@ module PostgresHaAdmin
       db_result.clear
       result
     rescue PG::Error => err
-      @logger.error("#{err.class}: #{err}")
-      @logger.error(err.backtrace.join("\n"))
+      logger.error("#{err.class}: #{err}")
+      logger.error(err.backtrace.join("\n"))
       result
     end
 
-    def entry_is_active_master?(record)
-      record[:type] == 'primary' && record[:active]
-    end
-
-    def all_databases
-      return [] unless File.exist?(yml_file)
-      YAML.load_file(yml_file)
-    end
-
     def table_exists?(connection, table_name)
       result = connection.exec("SELECT to_regclass('#{table_name}')").first
       !result['to_regclass'].nil?

data/lib/manageiq/postgres_ha_admin/version.rb

@@ -1,5 +1,5 @@
 module ManageIQ
   module PostgresHaAdmin
-    VERSION = '2.0.0'.freeze
+    VERSION = '3.0.0'.freeze
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: manageiq-postgres_ha_admin
 version: !ruby/object:Gem::Version
-  version: 2.0.0
+  version: 3.0.0
 platform: ruby
 authors:
 - ManageIQ Developers
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2018-08-01 00:00:00.000000000 Z
+date: 2018-09-05 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: awesome_spawn
@@ -152,8 +152,7 @@ dependencies:
         version: '0'
 description: ManageIQ Postgres H.A. Admin
 email: 
-executables:
-- postgres_ha_admin
+executables: []
 extensions: []
 extra_rdoc_files: []
 files:
@@ -170,11 +169,14 @@ files:
 - LICENSE.txt
 - README.md
 - Rakefile
-- bin/postgres_ha_admin
 - lib/manageiq-postgres_ha_admin.rb
-- lib/manageiq/postgres_ha_admin/database_yml.rb
-- lib/manageiq/postgres_ha_admin/failover_databases.rb
+- lib/manageiq/postgres_ha_admin/config_handler.rb
+- lib/manageiq/postgres_ha_admin/config_handler/pglogical_config_handler.rb
+- lib/manageiq/postgres_ha_admin/config_handler/rails_config_handler.rb
 - lib/manageiq/postgres_ha_admin/failover_monitor.rb
+- lib/manageiq/postgres_ha_admin/logging.rb
+- lib/manageiq/postgres_ha_admin/null_logger.rb
+- lib/manageiq/postgres_ha_admin/server_store.rb
 - lib/manageiq/postgres_ha_admin/version.rb
 - manageiq-postgres_ha_admin.gemspec
 homepage: https://github.com/ManageIQ/manageiq-postgres_ha_admin

data/bin/postgres_ha_admin

@@ -1,7 +0,0 @@
-#!/bin/env ruby
-
-require 'bundler/setup'
-require 'manageiq-postgres_ha_admin'
-
-monitor = ManageIQ::PostgresHaAdmin::FailoverMonitor.new
-monitor.monitor_loop