rails_failover 0.3.0 → 0.5.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 307117d6a7083a17a2bf2799737f6f9be3afa77a58bace4f867bedf38fa05a5c
-  data.tar.gz: 9d9d54a7a46785f259ba3f19aad402534de7b5305bd57993cb9c92fd14efad3b
+  metadata.gz: 31e5bf77f85686776c2cf344b2e145af8e99a984696102cbdf51b457d45f1668
+  data.tar.gz: aa6556a7bf399949066078bd463eeb9744db5e16a7152bf112a1ba023226b4bf
 SHA512:
-  metadata.gz: 7c45a722a9d8bbe1d0ea1410630252c5485496a49b6affe5461615779102d25bfafbf0d990bce920d8e52f164ebe1b92e94a1ccd1787d526b98cba54fc039d45
-  data.tar.gz: 3dcd9668ef0266b91c35cb3764533fab3624025a3d4dd6db86ce1adafb8168c9d72dde14bbd57fbcb73f7f0049941e6b077eaba519b70b3fa1b03eb92d730b64
+  metadata.gz: 46a47eab782fa173e3a18b8d0159798129f39739ffdc9e910f06ba698c6a78689df33120d67278b1b56d1dda0b83d25cbbcd4919ae2384eb932fbc33481f087f
+  data.tar.gz: aa69d48628da810cbefbd386a093cff557425d4e77f22985bcc7bbb75b25a522417b4d36a3ae9172832f880974614fd7056187b71e05049c605ec8d92e80de96

data/.rubocop.yml

@@ -1,3 +1,7 @@
+AllCops:
+  Exclude:
+    - spec/support/dummy_app/**/*
+
 inherit_gem:
   rubocop-discourse: default.yml
 

data/CHANGELOG.md

@@ -0,0 +1,12 @@
+# Changelog
+All notable changes to this project will be documented in this file.
+
+The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
+and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
+
+## [Unreleased]
+
+## [0.5.2] - 2020-06-23
+
+### Changed
+- FIX: Only rescue from connection errors.q

data/Gemfile

@@ -6,14 +6,9 @@ gemspec
 
 gem "rake", "~> 12.0"
 gem "rspec", "~> 3.0"
-
-group :development do
-  gem 'rubocop-discourse'
-end
-
-group :development, :test do
-  gem 'byebug'
-  gem 'redis', '~> 4.1'
-  gem 'pg', '~> 1.2'
-  gem 'activerecord', '~> 6.0'
-end
+gem 'rubocop-discourse'
+gem 'byebug'
+gem 'redis', '~> 4.1'
+gem 'pg', '~> 1.2'
+gem 'activerecord', '~> 6.0'
+gem 'rack'

data/Gemfile.lock

@@ -1,9 +1,8 @@
 PATH
   remote: .
   specs:
-    rails_failover (0.3.0)
+    rails_failover (0.5.3)
       activerecord (~> 6.0)
-      listen (~> 3.2)
       railties (~> 6.0)
 
 GEM
@@ -40,25 +39,21 @@ GEM
     crass (1.0.6)
     diff-lcs (1.3)
     erubi (1.9.0)
-    ffi (1.12.2)
     i18n (1.8.2)
       concurrent-ruby (~> 1.0)
-    listen (3.2.1)
-      rb-fsevent (~> 0.10, >= 0.10.3)
-      rb-inotify (~> 0.9, >= 0.9.10)
-    loofah (2.5.0)
+    loofah (2.6.0)
       crass (~> 1.0.2)
       nokogiri (>= 1.5.9)
     method_source (1.0.0)
     mini_portile2 (2.4.0)
     minitest (5.14.1)
-    nokogiri (1.10.9)
+    nokogiri (1.10.10)
       mini_portile2 (~> 2.4.0)
     parallel (1.19.1)
     parser (2.7.1.2)
       ast (~> 2.4.0)
     pg (1.2.3)
-    rack (2.2.2)
+    rack (2.2.3)
     rack-test (1.1.0)
       rack (>= 1.0, < 3)
     rails-dom-testing (2.0.3)
@@ -74,9 +69,6 @@ GEM
       thor (>= 0.20.3, < 2.0)
     rainbow (3.0.0)
     rake (12.3.3)
-    rb-fsevent (0.10.4)
-    rb-inotify (0.10.1)
-      ffi (~> 1.0)
     redis (4.1.4)
     rexml (3.2.4)
     rspec (3.9.0)
@@ -119,6 +111,7 @@ DEPENDENCIES
   activerecord (~> 6.0)
   byebug
   pg (~> 1.2)
+  rack
   rails_failover!
   rake (~> 12.0)
   redis (~> 4.1)

data/README.md

@@ -1,13 +1,16 @@
 # RailsFailover
 
-* Automatic failover and recovery for simple master-replica Redis setup
+Automatic failover and recovery for primary/replica setup for:
+
+1. Redis
+1. ActiveRecord (PostgreSQL/MySQL Adapters)
 
 ## Installation
 
 Add this line to your application's Gemfile:
 
 ```ruby
-gem 'rails_failover'
+gem 'rails_failover', require: false
 ```
 
 And then execute:
@@ -36,21 +39,58 @@ production:
 
 The gem will automatically create an `ActiveRecord::ConnectionAdapters::ConnectionHandler` with the `ActiveRecord::Base.reading_role` as the `handler_key`.
 
+#### Failover/Fallback Hooks
+
+```
+RailsFailover::ActiveRecord.on_failover do
+  # Enable readonly mode
+end
+
+RailsFailover::ActiveRecord.on_fallback do
+  # Disable readonly mode
+end
+```
+
+#### Multiple connection handlers
+
+Note: This API is unstable and is likely to changes when Rails 6.1 is released with sharding support.
+
+```
+# config/database.yml
+
+production:
+  primary:
+    host: <primary db server host>
+    port: <primary db server port>
+    replica_host: <replica db server host>
+    replica_port: <replica db server port>
+  second_database_writing:
+    host: <primary db server host>
+    port: <primary db server port>
+    replica_host: <replica db server host>
+    replica_port: <replica db server port>
+
+# In your ActiveRecord base model or model.
+
+connects_to database: { writing: :primary, second_database_writing: :second_database_writing
+```
+
 ### Redis
 
+Add `require 'rails_failover/redis'` before creating a `Redis` instance.
+
 ```
 Redis.new(host: "127.0.0.1", port: 6379, replica_host: "127.0.0.1", replica_port: 6380, connector: RailsFailover::Redis::Connector))
 ```
 
-Callbacks can be registered when the master connection is down and when it is up.
-
+Callbacks can be registered when the primary connection is down and when it is up.
 
 ```
-RailsFailover::Redis.register_master_up_callback do
+RailsFailover::Redis.on_failover_callback do
   # Switch site to read-only mode
 end
 
-RailsFailover::Redis.register_master_down_callback do
+RailsFailover::Redis.on_fallback_callback do
   # Switch site out of read-only mode
 end
 ```
@@ -79,7 +119,6 @@ The ActiveRecord failover tests are run against a dummy Rails server. Run the fo
 
 Bug reports and pull requests are welcome on GitHub at https://github.com/discourse/rails_failover. This project is intended to be a safe, welcoming space for collaboration, and contributors are expected to adhere to the [code of conduct](https://github.com/discourse/rails_failover/blob/master/CODE_OF_CONDUCT.md).
 
-
 ## License
 
 The gem is available as open source under the terms of the [MIT License](https://opensource.org/licenses/MIT).

data/lib/rails_failover/active_record.rb

@@ -1,7 +1,11 @@
 # frozen_string_literal: true
 
 require 'active_record'
-require_relative 'active_record/railtie'
+
+if defined?(::Rails)
+  require_relative 'active_record/railtie'
+end
+
 require_relative 'active_record/middleware'
 require_relative 'active_record/handler'
 
@@ -23,19 +27,36 @@ module RailsFailover
       @verify_primary_frequency_seconds || 5
     end
 
-    def self.establish_reading_connection(connection_spec)
+    def self.establish_reading_connection(handler, connection_spec)
       config = connection_spec.config
 
       if config[:replica_host] && config[:replica_port]
         replica_config = config.dup
-
         replica_config[:host] = replica_config.delete(:replica_host)
         replica_config[:port] = replica_config.delete(:replica_port)
         replica_config[:replica] = true
-
-        handler = ::ActiveRecord::Base.connection_handlers[::ActiveRecord::Base.reading_role]
         handler.establish_connection(replica_config)
       end
     end
+
+    def self.register_force_reading_role_callback(&block)
+      Middleware.force_reading_role_callback = block
+    end
+
+    def self.on_failover(&block)
+      @on_failover_callback = block
+    end
+
+    def self.on_failover_callback
+      @on_failover_callback
+    end
+
+    def self.on_fallback(&block)
+      @on_fallback_callback = block
+    end
+
+    def self.on_fallback_callback
+      @on_fallback_callback
+    end
   end
 end

data/lib/rails_failover/active_record/handler.rb

@@ -1,8 +1,6 @@
 # frozen_string_literal: true
 require 'singleton'
 require 'monitor'
-require 'listen'
-require 'fileutils'
 
 module RailsFailover
   module ActiveRecord
@@ -10,61 +8,45 @@ module RailsFailover
       include Singleton
       include MonitorMixin
 
-      SEPERATOR = "__RAILS_FAILOVER__"
       VERIFY_FREQUENCY_BUFFER_PRECENT = 20
 
       def initialize
         @primaries_down = {}
         @ancestor_pid = Process.pid
 
-        @dir = '/tmp/rails_failover'
-        FileUtils.remove_dir(@dir) if Dir.exists?(@dir)
-        FileUtils.mkdir_p(@dir)
-
-        @listener = Listen.to(@dir) do |modified, added, removed|
-          if added.length > 0
-            added.each do |f|
-              pid, handler_key = File.basename(f).split(SEPERATOR)
-
-              if Process.pid != pid
-                verify_primary(handler_key.to_sym, publish: false)
-              end
-            end
-          end
-
-          if removed.length > 0
-            removed.each do |f|
-              pid, handler_key = File.basename(f).split(SEPERATOR)
-
-              if Process.pid != pid
-                primary_up(handler_key.to_sym)
-              end
-            end
-          end
-        end
-
         super() # Monitor#initialize
       end
 
-      def start_listener
-        @listener.start
-      end
-
-      def verify_primary(handler_key, publish: true)
+      def verify_primary(handler_key)
         mon_synchronize do
           primary_down(handler_key)
-          publish_primary_down(handler_key) if publish
-          return if @thread&.alive? && @thread["pid"] == Process.pid
+          return if @thread&.alive?
+
+          logger.warn "Failover for ActiveRecord has been initiated"
+
+          begin
+            RailsFailover::ActiveRecord.on_failover_callback&.call
+          rescue => e
+            logger.warn("RailsFailover::ActiveRecord.on_failover_callback failed: #{e.class} #{e.message}\n#{e.backtrace.join("\n")}")
+          end
 
           @thread = Thread.new do
             loop do
               initiate_fallback_to_primary
-              break if all_primaries_up
+
+              if all_primaries_up
+                logger.warn "Fallback to primary for ActiveRecord has been completed."
+
+                begin
+                  RailsFailover::ActiveRecord.on_fallback_callback&.call
+                rescue => e
+                  logger.warn("RailsFailover::ActiveRecord.on_fallback_callback failed: #{e.class} #{e.message}\n#{e.backtrace.join("\n")}")
+                end
+
+                break
+              end
             end
           end
-
-          @thread["pid"] = Process.pid
-          @thread
         end
       end
 
@@ -78,27 +60,26 @@ module RailsFailover
           connection_handler = ::ActiveRecord::Base.connection_handlers[handler_key]
           spec = connection_handler.retrieve_connection_pool(spec_name).spec
           config = spec.config
-          logger.warn "#{Process.pid} Checking server for '#{handler_key} #{spec_name}'..."
+          logger.debug "#{Process.pid} Checking server for '#{handler_key} #{spec_name}'..."
           connection_active = false
 
           begin
             connection = ::ActiveRecord::Base.public_send(spec.adapter_method, config)
             connection_active = connection.active?
           rescue => e
-            logger.warn "#{Process.pid} Connection to server for '#{handler_key} #{spec_name}' failed with '#{e.message}'"
+            logger.debug "#{Process.pid} Connection to server for '#{handler_key} #{spec_name}' failed with '#{e.message}'"
           ensure
             connection.disconnect! if connection
           end
 
           if connection_active
-            logger.warn "#{Process.pid} Server for '#{handler_key} #{spec_name}' is active."
+            logger.debug "#{Process.pid} Server for '#{handler_key} #{spec_name}' is active."
             active_handler_keys << handler_key
           end
         end
 
         active_handler_keys.each do |handler_key|
           primary_up(handler_key)
-          publish_primary_up(handler_key)
         end
       end
 
@@ -114,25 +95,12 @@ module RailsFailover
         end
       end
 
-
       def primary_down(handler_key)
         mon_synchronize do
           primaries_down[handler_key] = true
         end
       end
 
-      def publish_primary_down(handler_key)
-        FileUtils.touch("#{@dir}/#{Process.pid}#{SEPERATOR}#{handler_key}")
-      end
-
-      def publish_primary_up(handler_key)
-        path = "#{@dir}/#{Process.pid}#{SEPERATOR}#{handler_key}"
-
-        if File.exists?(path)
-          FileUtils.rm("#{@dir}/#{Process.pid}#{SEPERATOR}#{handler_key}")
-        end
-      end
-
       def primary_up(handler_key)
         mon_synchronize do
           primaries_down.delete(handler_key)
@@ -165,7 +133,7 @@ module RailsFailover
       end
 
       def logger
-        Rails.logger
+        ::Rails.logger
       end
     end
   end

data/lib/rails_failover/active_record/middleware.rb

@@ -2,39 +2,100 @@
 
 module RailsFailover
   module ActiveRecord
+    class Interceptor
+      def self.adapter_errors
+        @adapter_errors ||= begin
+          if defined?(::PG)
+            [::PG::UnableToSend, ::PG::ConnectionBad]
+          elsif defined?(::Mysql2)
+            [::Mysql2::Error::ConnectionError]
+          end
+        end
+      end
+
+      def self.handle(request, exception)
+        exception = resolve_cause(exception)
+
+        if adapter_errors.any? { |error| exception.is_a?(error) }
+          Handler.instance.verify_primary(request.env[Middleware::WRITING_ROLE_HEADER])
+        end
+      end
+
+      def self.resolve_cause(exception)
+        if exception.cause
+          resolve_cause(exception.cause)
+        else
+          exception
+        end
+      end
+    end
+
     class Middleware
+      class << self
+        attr_accessor :force_reading_role_callback
+      end
+
+      CURRENT_ROLE_HEADER = "rails_failover.role"
+      WRITING_ROLE_HEADER = "rails_failover.writing_role"
+
       def initialize(app)
         @app = app
       end
 
       def call(env)
-        writing_role = ::ActiveRecord::Base.writing_role
+        current_role = ::ActiveRecord::Base.current_role || ::ActiveRecord::Base.writing_role
+        is_writing_role = current_role.to_s.end_with?(::ActiveRecord::Base.writing_role.to_s)
+        writing_role = resolve_writing_role(current_role, is_writing_role)
 
         role =
-          if primary_down = Handler.instance.primary_down?(writing_role)
-            ::ActiveRecord::Base.reading_role
+          if primary_down = self.class.force_reading_role_callback&.call(env) || Handler.instance.primary_down?(writing_role)
+            reading_role = resolve_reading_role(current_role, is_writing_role)
+            ensure_reading_connection_established!(writing_role: writing_role, reading_role: reading_role)
+            reading_role
           else
-            ::ActiveRecord::Base.writing_role
+            writing_role
           end
 
         ::ActiveRecord::Base.connected_to(role: role) do
-          env["rails_failover.role"] = role
+          env[CURRENT_ROLE_HEADER] = role
+          env[WRITING_ROLE_HEADER] = writing_role
           @app.call(env)
         end
-      rescue Exception => e
-        if (resolve_cause(e).is_a?(::PG::Error))
-          Handler.instance.verify_primary(writing_role)
-          raise
-        end
       end
 
       private
 
-      def resolve_cause(error)
-        if error.cause
-          resolve_cause(error.cause)
+      def ensure_reading_connection_established!(writing_role:, reading_role:)
+        ::ActiveRecord::Base.connection_handlers[reading_role] ||= begin
+          handler = ::ActiveRecord::ConnectionAdapters::ConnectionHandler.new
+
+          ::ActiveRecord::Base.connection_handlers[writing_role].connection_pools.each do |pool|
+            ::RailsFailover::ActiveRecord.establish_reading_connection(handler, pool.spec)
+          end
+
+          handler
+        end
+      end
+
+      def resolve_writing_role(current_role, is_writing_role)
+        if is_writing_role
+          current_role
+        else
+          current_role.to_s.sub(
+            /#{::ActiveRecord::Base.reading_role}$/,
+            ::ActiveRecord::Base.writing_role.to_s
+          ).to_sym
+        end
+      end
+
+      def resolve_reading_role(current_role, is_writing_role)
+        if is_writing_role
+          current_role.to_s.sub(
+            /#{::ActiveRecord::Base.writing_role}$/,
+            ::ActiveRecord::Base.reading_role.to_s
+          ).to_sym
         else
-          error
+          current_role
         end
       end
     end

data/lib/rails_failover/active_record/railtie.rb

@@ -1,35 +1,57 @@
+# frozen_string_literal: true
+
 module RailsFailover
   module ActiveRecord
     class Railtie < ::Rails::Railtie
-      initializer "rails_failover.init", after: "active_record.initialize_database" do
-        ::ActiveSupport.on_load(:active_record) do
-          Handler.instance
+      initializer "rails_failover.init", after: "active_record.initialize_database" do |app|
+        config = ::ActiveRecord::Base.connection_config
+        app.config.active_record_rails_failover = false
 
-          # We are doing this manually for now since we're awaiting Rails 6.1 to be released which will
-          # have more stable ActiveRecord APIs for handling multiple databases with different roles.
-          ::ActiveRecord::Base.connection_handlers[::ActiveRecord::Base.reading_role] =
-            ::ActiveRecord::ConnectionAdapters::ConnectionHandler.new
+        if !!(config[:replica_host] && config[:replica_port])
+          app.config.active_record_rails_failover = true
 
-          ::ActiveRecord::Base.connection_handlers[::ActiveRecord::Base.writing_role].connection_pools.each do |connection_pool|
-            RailsFailover::ActiveRecord.establish_reading_connection(connection_pool.spec)
-          end
+          ::ActiveSupport.on_load(:active_record) do
+            Handler.instance
+
+            # We are doing this manually for now since we're awaiting Rails 6.1 to be released which will
+            # have more stable ActiveRecord APIs for handling multiple databases with different roles.
+            ::ActiveRecord::Base.connection_handlers[::ActiveRecord::Base.reading_role] =
+              ::ActiveRecord::ConnectionAdapters::ConnectionHandler.new
 
-          begin
-            ::ActiveRecord::Base.connection
-          rescue ::ActiveRecord::NoDatabaseError
-            # Do nothing since database hasn't been created
-          rescue ::PG::Error => e
-            Handler.instance.verify_primary(::ActiveRecord::Base.writing_role)
-            ::ActiveRecord::Base.connection_handler = ::ActiveRecord::Base.lookup_connection_handler(:reading)
+            ::ActiveRecord::Base.connection_handlers[::ActiveRecord::Base.writing_role].connection_pools.each do |connection_pool|
+              RailsFailover::ActiveRecord.establish_reading_connection(
+                ::ActiveRecord::Base.connection_handlers[::ActiveRecord::Base.reading_role],
+                connection_pool.spec
+              )
+            end
+
+            begin
+              ::ActiveRecord::Base.connection
+            rescue ::ActiveRecord::NoDatabaseError
+              # Do nothing since database hasn't been created
+            rescue ::PG::Error => e
+              Handler.instance.verify_primary(::ActiveRecord::Base.writing_role)
+              ::ActiveRecord::Base.connection_handler = ::ActiveRecord::Base.lookup_connection_handler(:reading)
+            end
           end
         end
       end
 
       initializer "rails_failover.insert_middleware" do |app|
-        app.middleware.insert_after(
-          ::ActionDispatch::ActionableExceptions,
-          ::RailsFailover::ActiveRecord::Middleware
-        )
+        if app.config.active_record_rails_failover
+          ActionDispatch::DebugExceptions.register_interceptor do |request, exception|
+            RailsFailover::ActiveRecord::Interceptor.handle(request, exception)
+          end
+
+          if !skip_middleware?(app.config)
+            app.middleware.unshift(::RailsFailover::ActiveRecord::Middleware)
+          end
+        end
+      end
+
+      def skip_middleware?(config)
+        return false if !config.respond_to?(:skip_rails_failover_active_record_middleware)
+        config.skip_rails_failover_active_record_middleware
       end
     end
   end

data/lib/rails_failover/redis.rb

@@ -18,33 +18,41 @@ module RailsFailover
     end
 
     def self.logger
-      @logger
+      if @logger
+        @logger
+      elsif defined?(::Rails)
+        ::Rails.logger
+      end
     end
 
-    def self.verify_master_frequency_seconds=(seconds)
-      @verify_master_frequency_seconds = seconds
+    def self.verify_primary_frequency_seconds=(seconds)
+      @verify_primary_frequency_seconds = seconds
     end
 
-    def self.verify_master_frequency_seconds
-      @verify_master_frequency_seconds || 5
+    def self.verify_primary_frequency_seconds
+      @verify_primary_frequency_seconds || 5
     end
 
-    def self.register_master_down_callback(&block)
-      @master_down_callbacks ||= []
-      @master_down_callbacks.push(block)
+    def self.on_failover(&block)
+      @on_failover_callback = block
     end
 
-    def self.master_down_callbacks
-      @master_down_callbacks || []
+    def self.on_failover_callback
+      @on_failover_callback
     end
 
-    def self.register_master_up_callback(&block)
-      @master_up_callbacks ||= []
-      @master_up_callbacks.push(block)
+    def self.on_fallback(&block)
+      @on_fallback_callback = block
     end
 
-    def self.master_up_callbacks
-      @master_up_callbacks || []
+    def self.on_fallback_callback
+      @on_fallback_callback
+    end
+
+    # For testing
+    def self.clear_callbacks
+      @on_fallback_callback = nil
+      @on_failover_callback = nil
     end
   end
 end

data/lib/rails_failover/redis/connector.rb

@@ -6,7 +6,7 @@ module RailsFailover
   class Redis
     class Connector < ::Redis::Client::Connector
       def initialize(options)
-        options[:original_driver] = options[:driver]
+        orignal_driver = options[:driver]
 
         options[:driver] = Class.new(options[:driver]) do
           def self.connect(options)
@@ -22,18 +22,24 @@ module RailsFailover
                  Errno::ETIMEDOUT,
                  Errno::EINVAL => e
 
-            Handler.instance.verify_master(options.dup)
+            Handler.instance.verify_primary(options)
             raise e
           end
         end
 
+        options[:original_driver] = orignal_driver
         options.delete(:connector)
-        @options = options.dup
+        options[:id] ||= "#{options[:host]}:#{options[:port]}"
         @replica_options = replica_options(options)
+        @options = options.dup
       end
 
       def resolve
-        Handler.instance.master ? @options : @replica_options
+        if Handler.instance.primary_down?(@options)
+          @replica_options
+        else
+          @options
+        end
       end
 
       def check(client)
@@ -50,7 +56,6 @@ module RailsFailover
         opts = options.dup
         opts[:host] = opts.delete(:replica_host)
         opts[:port] = opts.delete(:replica_port)
-        opts[:driver] = opts.delete(:original_driver)
         opts
       end
     end

data/lib/rails_failover/redis/handler.rb

@@ -9,100 +9,184 @@ module RailsFailover
       include Singleton
       include MonitorMixin
 
-      MASTER_ROLE_STATUS = "role:master"
-      MASTER_LOADED_STATUS = "loading:0"
+      PRIMARY_ROLE_STATUS = "role:master"
+      PRIMARY_LOADED_STATUS = "loading:0"
+      VERIFY_FREQUENCY_BUFFER_PRECENT = 20
 
       def initialize
-        @master = true
-        @clients = []
+        @primaries_down = {}
+        @clients = {}
+        @ancestor_pid = Process.pid
 
         super() # Monitor#initialize
       end
 
-      def verify_master(options)
+      def verify_primary(options)
         mon_synchronize do
+          primary_down(options)
+          disconnect_clients(options)
+
           return if @thread&.alive?
 
-          self.master = false
-          disconnect_clients
-          RailsFailover::Redis.master_down_callbacks.each { |callback| callback.call }
+          logger&.warn "Failover for Redis has been initiated"
+
+          begin
+            RailsFailover::Redis.on_failover_callback&.call
+          rescue => e
+            logger&.warn("RailsFailover::Redis.on_failover_callback failed: #{e.class} #{e.message}\n#{e.backtrace.join("\n")}")
+          end
 
           @thread = Thread.new do
             loop do
-              thread = Thread.new { initiate_fallback_to_master(options) }
+              thread = Thread.new { initiate_fallback_to_primary }
               thread.join
-              break if self.master
-              sleep (RailsFailover::Redis.verify_master_frequency_seconds + (Time.now.to_i % RailsFailover::Redis.verify_master_frequency_seconds))
-            ensure
-              thread.kill
+
+              if all_primaries_up
+                logger&.warn "Fallback to primary for Redis has been completed."
+
+                begin
+                  RailsFailover::Redis.on_fallback_callback&.call
+                rescue => e
+                  logger&.warn("RailsFailover::Redis.on_fallback_callback failed: #{e.class} #{e.message}\n#{e.backtrace.join("\n")}")
+                end
+
+                break
+              end
             end
           end
         end
       end
 
-      def initiate_fallback_to_master(options)
-        info = nil
-
-        begin
-          master_client = ::Redis::Client.new(options.dup)
-          log "#{log_prefix}: Checking connection to master server..."
-          info = master_client.call([:info])
-        rescue => e
-          log "#{log_prefix}: Connection to Master server failed with '#{e.message}'"
-        ensure
-          master_client&.disconnect
+      def initiate_fallback_to_primary
+        frequency = RailsFailover::Redis.verify_primary_frequency_seconds
+        sleep(frequency * ((rand(VERIFY_FREQUENCY_BUFFER_PRECENT) + 100) / 100.0))
+
+        active_primaries_keys = {}
+
+        primaries_down.each do |key, options|
+          info = nil
+          options = options.dup
+
+          begin
+            options[:driver] = options[:original_driver]
+            primary_client = ::Redis::Client.new(options)
+            logger&.debug "Checking connection to primary server (#{key})"
+            info = primary_client.call([:info])
+          rescue => e
+            logger&.debug "Connection to primary server (#{key}) failed with '#{e.message}'"
+          ensure
+            primary_client&.disconnect
+          end
+
+          if info && info.include?(PRIMARY_LOADED_STATUS) && info.include?(PRIMARY_ROLE_STATUS)
+            active_primaries_keys[key] = options
+            logger&.debug "Primary server (#{key}) is active, disconnecting clients from replica"
+          end
         end
 
-        if info && info.include?(MASTER_LOADED_STATUS) && info.include?(MASTER_ROLE_STATUS)
-          self.master = true
-          log "#{log_prefix}: Master server is active, disconnecting clients from replica"
-          disconnect_clients
-          RailsFailover::Redis.master_up_callbacks.each { |callback| callback.call }
+        active_primaries_keys.each do |key, options|
+          primary_up(options)
+          disconnect_clients(options)
         end
       end
 
       def register_client(client)
+        key = client.options[:id]
+
         mon_synchronize do
-          @clients << client
+          clients[key] ||= []
+          clients[key] << client
         end
       end
 
       def deregister_client(client)
+        key = client.options[:id]
+
         mon_synchronize do
-          @clients.delete(client)
+          if clients[key]
+            clients[key].delete(client)
+
+            if clients[key].empty?
+              clients.delete(key)
+            end
+          end
         end
       end
 
-      def clients
-        mon_synchronize { @clients }
+      def primary_down?(options)
+        mon_synchronize do
+          primaries_down[options[:id]]
+        end
       end
 
-      def master
-        mon_synchronize { @master }
+      private
+
+      def all_primaries_up
+        mon_synchronize { primaries_down.empty? }
       end
 
-      def master=(args)
-        mon_synchronize { @master = args }
+      def primary_up(options)
+        mon_synchronize do
+          primaries_down.delete(options[:id])
+        end
       end
 
-      private
+      def primary_down(options)
+        mon_synchronize do
+          primaries_down[options[:id]] = options.dup
+        end
+      end
+
+      def clients
+        process_pid = Process.pid
+        return @clients[process_pid] if @clients[process_pid]
 
-      def disconnect_clients
         mon_synchronize do
-          @clients.dup.each do |c|
-            c.disconnect
+          if !@clients[process_pid]
+            @clients[process_pid] = {}
+
+            if process_pid != @ancestor_pid
+              @clients.delete(@ancestor_pid)
+            end
           end
+
+          @clients[process_pid]
         end
       end
 
-      def log(message)
-        if logger = RailsFailover::Redis.logger
-          logger.warn(message)
+      def primaries_down
+        process_pid = Process.pid
+        return @primaries_down[process_pid] if @primaries_down[process_pid]
+
+        mon_synchronize do
+          if !@primaries_down[process_pid]
+            @primaries_down[process_pid] = @primaries_down[@ancestor_pid] || {}
+
+            if process_pid != @ancestor_pid
+              @primaries_down.delete(@ancestor_pid).each do |id, options|
+                verify_primary(options)
+              end
+            end
+          end
+
+          @primaries_down[process_pid]
+        end
+      end
+
+      def disconnect_clients(options)
+        key = options[:id]
+
+        mon_synchronize do
+          if clients[key]
+            clients[key].dup.each do |c|
+              c.disconnect
+            end
+          end
         end
       end
 
-      def log_prefix
-        "#{self.class}"
+      def logger
+        RailsFailover::Redis.logger
       end
     end
   end

data/lib/rails_failover/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module RailsFailover
-  VERSION = "0.3.0"
+  VERSION = "0.5.3"
 end

data/makefile

@@ -12,7 +12,7 @@ setup_dummy_rails_server:
 	@cd spec/support/dummy_app && bundle install --quiet --without test --without development && yarn install && RAILS_ENV=production $(BUNDLER_BIN) exec rails db:create db:migrate db:seed
 
 start_dummy_rails_server:
-	@cd spec/support/dummy_app && BUNDLE_GEMFILE=Gemfile bundle exec unicorn -c config/unicorn.conf.rb -D -E production
+	@cd spec/support/dummy_app && BUNDLE_GEMFILE=Gemfile UNICORN_WORKERS=5 SECRET_KEY_BASE=somekey bundle exec unicorn -c config/unicorn.conf.rb -D -E production
 
 stop_dummy_rails_server:
 	@kill -TERM $(shell cat spec/support/dummy_app/tmp/pids/unicorn.pid)

data/rails_failover.gemspec

@@ -21,8 +21,6 @@ Gem::Specification.new do |spec|
   spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
   spec.require_paths = ["lib"]
 
-  spec.add_dependency 'listen', "~> 3.2"
-
   ["activerecord", "railties"].each do |gem_name|
     spec.add_dependency gem_name, "~> 6.0"
   end

data/redis.mk

@@ -1,7 +1,7 @@
 REDIS_PORT                 := 6381
 REDIS_PID_PATH             := /tmp/redis.pid
 REDIS_SOCKET_PATH          := /tmp/redis.sock
-REDIS_DBFILENAME           := master.rdb
+REDIS_DBFILENAME           := primary.rdb
 REDIS_REPLICA_PORT         := 6382
 REDIS_REPLICA_PID_PATH     := /tmp/redis_replica.pid
 REDIS_REPLICA_SOCKET_PATH  := /tmp/redis_replica.sock
@@ -12,17 +12,17 @@ redis: start_redis test_redis stop_redis
 test_redis:
 	@REDIS=1 bundle exec rspec --tag type:redis ${RSPEC_PATH}
 
-start_redis: start_redis_master start_redis_replica
-stop_redis: stop_redis_replica stop_redis_master
+start_redis: start_redis_primary start_redis_replica
+stop_redis: stop_redis_replica stop_redis_primary
 
-stop_redis_master:
+stop_redis_primary:
 	@redis-cli -p ${REDIS_PORT} shutdown
 
-start_redis_master:
+start_redis_primary:
 	@redis-server --daemonize yes --pidfile ${REDIS_PID_PATH} --port ${REDIS_PORT} --unixsocket ${REDIS_SOCKET_PATH} --dbfilename ${REDIS_DBFILENAME} --logfile /dev/null
 
 stop_redis_replica:
 	@redis-cli -p ${REDIS_REPLICA_PORT} shutdown
 
 start_redis_replica:
-	@redis-server --daemonize yes --pidfile ${REDIS_REPLICA_PID_PATH} --port ${REDIS_REPLICA_PORT} --unixsocket ${REDIS_REPLICA_SOCKET_PATH} --replicaof 127.0.0.1 ${REDIS_PORT} --dbfilename ${REDIS_REPLICA_DBFILENAME} --logfile /dev/null
+	@redis-server --daemonize yes --pidfile ${REDIS_REPLICA_PID_PATH} --port ${REDIS_REPLICA_PORT} --unixsocket ${REDIS_REPLICA_SOCKET_PATH} --slaveof 127.0.0.1 ${REDIS_PORT} --dbfilename ${REDIS_REPLICA_DBFILENAME} --logfile /dev/null

metadata

@@ -1,29 +1,15 @@
 --- !ruby/object:Gem::Specification
 name: rails_failover
 version: !ruby/object:Gem::Version
-  version: 0.3.0
+  version: 0.5.3
 platform: ruby
 authors:
 - Alan Tan
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2020-05-28 00:00:00.000000000 Z
+date: 2020-07-20 00:00:00.000000000 Z
 dependencies:
-- !ruby/object:Gem::Dependency
-  name: listen
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '3.2'
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '3.2'
 - !ruby/object:Gem::Dependency
   name: activerecord
   requirement: !ruby/object:Gem::Requirement
@@ -63,6 +49,7 @@ files:
 - ".rspec"
 - ".rubocop.yml"
 - ".travis.yml"
+- CHANGELOG.md
 - CODE_OF_CONDUCT.md
 - Gemfile
 - Gemfile.lock
@@ -105,7 +92,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.0.3
+rubygems_version: 3.1.2
 signing_key: 
 specification_version: 4
 summary: Failover for ActiveRecord and Redis