rails_failover 0.2.0 → 0.5.2

data/lib/rails_failover/active_record/middleware.rb

@@ -0,0 +1,103 @@
+# frozen_string_literal: true
+
+module RailsFailover
+  module ActiveRecord
+    class Interceptor
+      def self.adapter_errors
+        @adapter_errors ||= begin
+          if defined?(::PG)
+            [::PG::ServerError, ::PG::UnableToSend, ::PG::ConnectionBad]
+          elsif defined?(::Mysql2)
+            [::Mysql2::Error::ConnectionError]
+          end
+        end
+      end
+
+      def self.handle(request, exception)
+        exception = resolve_cause(exception)
+
+        if adapter_errors.any? { |error| exception.is_a?(error) }
+          Handler.instance.verify_primary(request.env[Middleware::WRITING_ROLE_HEADER])
+        end
+      end
+
+      def self.resolve_cause(exception)
+        if exception.cause
+          resolve_cause(exception.cause)
+        else
+          exception
+        end
+      end
+    end
+
+    class Middleware
+      class << self
+        attr_accessor :force_reading_role_callback
+      end
+
+      CURRENT_ROLE_HEADER = "rails_failover.role"
+      WRITING_ROLE_HEADER = "rails_failover.writing_role"
+
+      def initialize(app)
+        @app = app
+      end
+
+      def call(env)
+        current_role = ::ActiveRecord::Base.current_role || ::ActiveRecord::Base.writing_role
+        is_writing_role = current_role.to_s.end_with?(::ActiveRecord::Base.writing_role.to_s)
+        writing_role = resolve_writing_role(current_role, is_writing_role)
+
+        role =
+          if primary_down = self.class.force_reading_role_callback&.call(env) || Handler.instance.primary_down?(writing_role)
+            reading_role = resolve_reading_role(current_role, is_writing_role)
+            ensure_reading_connection_established!(writing_role: writing_role, reading_role: reading_role)
+            reading_role
+          else
+            writing_role
+          end
+
+        ::ActiveRecord::Base.connected_to(role: role) do
+          env[CURRENT_ROLE_HEADER] = role
+          env[WRITING_ROLE_HEADER] = writing_role
+          @app.call(env)
+        end
+      end
+
+      private
+
+      def ensure_reading_connection_established!(writing_role:, reading_role:)
+        ::ActiveRecord::Base.connection_handlers[reading_role] ||= begin
+          handler = ::ActiveRecord::ConnectionAdapters::ConnectionHandler.new
+
+          ::ActiveRecord::Base.connection_handlers[writing_role].connection_pools.each do |pool|
+            ::RailsFailover::ActiveRecord.establish_reading_connection(handler, pool.spec)
+          end
+
+          handler
+        end
+      end
+
+      def resolve_writing_role(current_role, is_writing_role)
+        if is_writing_role
+          current_role
+        else
+          current_role.to_s.sub(
+            /#{::ActiveRecord::Base.reading_role}$/,
+            ::ActiveRecord::Base.writing_role.to_s
+          ).to_sym
+        end
+      end
+
+      def resolve_reading_role(current_role, is_writing_role)
+        if is_writing_role
+          current_role.to_s.sub(
+            /#{::ActiveRecord::Base.writing_role}$/,
+            ::ActiveRecord::Base.reading_role.to_s
+          ).to_sym
+        else
+          current_role
+        end
+      end
+    end
+  end
+end

data/lib/rails_failover/active_record/railtie.rb

@@ -0,0 +1,58 @@
+# frozen_string_literal: true
+
+module RailsFailover
+  module ActiveRecord
+    class Railtie < ::Rails::Railtie
+      initializer "rails_failover.init", after: "active_record.initialize_database" do |app|
+        config = ::ActiveRecord::Base.connection_config
+        app.config.active_record_rails_failover = false
+
+        if !!(config[:replica_host] && config[:replica_port])
+          app.config.active_record_rails_failover = true
+
+          ::ActiveSupport.on_load(:active_record) do
+            Handler.instance
+
+            # We are doing this manually for now since we're awaiting Rails 6.1 to be released which will
+            # have more stable ActiveRecord APIs for handling multiple databases with different roles.
+            ::ActiveRecord::Base.connection_handlers[::ActiveRecord::Base.reading_role] =
+              ::ActiveRecord::ConnectionAdapters::ConnectionHandler.new
+
+            ::ActiveRecord::Base.connection_handlers[::ActiveRecord::Base.writing_role].connection_pools.each do |connection_pool|
+              RailsFailover::ActiveRecord.establish_reading_connection(
+                ::ActiveRecord::Base.connection_handlers[::ActiveRecord::Base.reading_role],
+                connection_pool.spec
+              )
+            end
+
+            begin
+              ::ActiveRecord::Base.connection
+            rescue ::ActiveRecord::NoDatabaseError
+              # Do nothing since database hasn't been created
+            rescue ::PG::Error => e
+              Handler.instance.verify_primary(::ActiveRecord::Base.writing_role)
+              ::ActiveRecord::Base.connection_handler = ::ActiveRecord::Base.lookup_connection_handler(:reading)
+            end
+          end
+        end
+      end
+
+      initializer "rails_failover.insert_middleware" do |app|
+        if app.config.active_record_rails_failover
+          ActionDispatch::DebugExceptions.register_interceptor do |request, exception|
+            RailsFailover::ActiveRecord::Interceptor.handle(request, exception)
+          end
+
+          if !skip_middleware?(app.config)
+            app.middleware.unshift(::RailsFailover::ActiveRecord::Middleware)
+          end
+        end
+      end
+
+      def skip_middleware?(config)
+        return false if !config.respond_to?(:skip_rails_failover_active_record_middleware)
+        config.skip_rails_failover_active_record_middleware
+      end
+    end
+  end
+end

data/lib/rails_failover/redis.rb

@@ -1,5 +1,14 @@
 # frozen_string_literal: true
 
+require 'redis'
+
+supported_version = '4'
+
+if Gem::Version.new(Redis::VERSION) < Gem::Version.new(supported_version)
+  raise "redis #{Redis::VERSION} is not supported. Please upgrade to Redis #{supported_version}."
+end
+
+require_relative "../redis/patches/client"
 require_relative 'redis/connector'
 
 module RailsFailover
@@ -9,33 +18,41 @@ module RailsFailover
     end
 
     def self.logger
-      @logger
+      if @logger
+        @logger
+      elsif defined?(::Rails)
+        ::Rails.logger
+      end
+    end
+
+    def self.verify_primary_frequency_seconds=(seconds)
+      @verify_primary_frequency_seconds = seconds
     end
 
-    def self.verify_master_frequency_seconds=(seconds)
-      @verify_master_frequency_seconds = seconds
+    def self.verify_primary_frequency_seconds
+      @verify_primary_frequency_seconds || 5
     end
 
-    def self.verify_master_frequency_seconds
-      @verify_master_frequency_seconds || 5
+    def self.on_failover(&block)
+      @on_failover_callback = block
     end
 
-    def self.register_master_down_callback(&block)
-      @master_down_callbacks ||= []
-      @master_down_callbacks.push(block)
+    def self.on_failover_callback
+      @on_failover_callback
     end
 
-    def self.master_down_callbacks
-      @master_down_callbacks || []
+    def self.on_fallback(&block)
+      @on_fallback_callback = block
     end
 
-    def self.register_master_up_callback(&block)
-      @master_up_callbacks ||= []
-      @master_up_callbacks.push(block)
+    def self.on_fallback_callback
+      @on_fallback_callback
     end
 
-    def self.master_up_callbacks
-      @master_up_callbacks || []
+    # For testing
+    def self.clear_callbacks
+      @on_fallback_callback = nil
+      @on_failover_callback = nil
     end
   end
 end

data/lib/rails_failover/redis/connector.rb

@@ -1,12 +1,12 @@
 # frozen_string_literal: true
 
-require_relative 'failover_handler'
+require_relative 'handler'
 
 module RailsFailover
   class Redis
-    class Connector
+    class Connector < ::Redis::Client::Connector
       def initialize(options)
-        options[:original_driver] = options[:driver]
+        orignal_driver = options[:driver]
 
         options[:driver] = Class.new(options[:driver]) do
           def self.connect(options)
@@ -22,26 +22,32 @@ module RailsFailover
                  Errno::ETIMEDOUT,
                  Errno::EINVAL => e
 
-            FailoverHandler.instance.verify_master(options.dup)
+            Handler.instance.verify_primary(options)
             raise e
           end
         end
 
+        options[:original_driver] = orignal_driver
         options.delete(:connector)
-        @options = options.dup
+        options[:id] ||= "#{options[:host]}:#{options[:port]}"
         @replica_options = replica_options(options)
+        @options = options.dup
       end
 
       def resolve
-        FailoverHandler.instance.master ? @options : @replica_options
+        if Handler.instance.primary_down?(@options)
+          @replica_options
+        else
+          @options
+        end
       end
 
       def check(client)
-        FailoverHandler.instance.register_client(client)
+        Handler.instance.register_client(client)
       end
 
       def on_disconnect(client)
-        FailoverHandler.instance.deregister_client(client)
+        Handler.instance.deregister_client(client)
       end
 
       private
@@ -50,7 +56,6 @@ module RailsFailover
         opts = options.dup
         opts[:host] = opts.delete(:replica_host)
         opts[:port] = opts.delete(:replica_port)
-        opts[:driver] = opts.delete(:original_driver)
         opts
       end
     end

data/lib/rails_failover/redis/handler.rb

@@ -0,0 +1,193 @@
+# frozen_string_literal: true
+
+require 'monitor'
+require 'singleton'
+
+module RailsFailover
+  class Redis
+    class Handler
+      include Singleton
+      include MonitorMixin
+
+      PRIMARY_ROLE_STATUS = "role:master"
+      PRIMARY_LOADED_STATUS = "loading:0"
+      VERIFY_FREQUENCY_BUFFER_PRECENT = 20
+
+      def initialize
+        @primaries_down = {}
+        @clients = {}
+        @ancestor_pid = Process.pid
+
+        super() # Monitor#initialize
+      end
+
+      def verify_primary(options)
+        mon_synchronize do
+          primary_down(options)
+          disconnect_clients(options)
+
+          return if @thread&.alive?
+
+          logger&.warn "Failover for Redis has been initiated"
+
+          begin
+            RailsFailover::Redis.on_failover_callback&.call
+          rescue => e
+            logger&.warn("RailsFailover::Redis.on_failover_callback failed: #{e.class} #{e.message}\n#{e.backtrace.join("\n")}")
+          end
+
+          @thread = Thread.new do
+            loop do
+              thread = Thread.new { initiate_fallback_to_primary }
+              thread.join
+
+              if all_primaries_up
+                logger&.warn "Fallback to primary for Redis has been completed."
+
+                begin
+                  RailsFailover::Redis.on_fallback_callback&.call
+                rescue => e
+                  logger&.warn("RailsFailover::Redis.on_fallback_callback failed: #{e.class} #{e.message}\n#{e.backtrace.join("\n")}")
+                end
+
+                break
+              end
+            end
+          end
+        end
+      end
+
+      def initiate_fallback_to_primary
+        frequency = RailsFailover::Redis.verify_primary_frequency_seconds
+        sleep(frequency * ((rand(VERIFY_FREQUENCY_BUFFER_PRECENT) + 100) / 100.0))
+
+        active_primaries_keys = {}
+
+        primaries_down.each do |key, options|
+          info = nil
+          options = options.dup
+
+          begin
+            options[:driver] = options[:original_driver]
+            primary_client = ::Redis::Client.new(options)
+            logger&.debug "Checking connection to primary server (#{key})"
+            info = primary_client.call([:info])
+          rescue => e
+            logger&.debug "Connection to primary server (#{key}) failed with '#{e.message}'"
+          ensure
+            primary_client&.disconnect
+          end
+
+          if info && info.include?(PRIMARY_LOADED_STATUS) && info.include?(PRIMARY_ROLE_STATUS)
+            active_primaries_keys[key] = options
+            logger&.debug "Primary server (#{key}) is active, disconnecting clients from replica"
+          end
+        end
+
+        active_primaries_keys.each do |key, options|
+          primary_up(options)
+          disconnect_clients(options)
+        end
+      end
+
+      def register_client(client)
+        key = client.options[:id]
+
+        mon_synchronize do
+          clients[key] ||= []
+          clients[key] << client
+        end
+      end
+
+      def deregister_client(client)
+        key = client.options[:id]
+
+        mon_synchronize do
+          if clients[key]
+            clients[key].delete(client)
+
+            if clients[key].empty?
+              clients.delete(key)
+            end
+          end
+        end
+      end
+
+      def primary_down?(options)
+        mon_synchronize do
+          primaries_down[options[:id]]
+        end
+      end
+
+      private
+
+      def all_primaries_up
+        mon_synchronize { primaries_down.empty? }
+      end
+
+      def primary_up(options)
+        mon_synchronize do
+          primaries_down.delete(options[:id])
+        end
+      end
+
+      def primary_down(options)
+        mon_synchronize do
+          primaries_down[options[:id]] = options.dup
+        end
+      end
+
+      def clients
+        process_pid = Process.pid
+        return @clients[process_pid] if @clients[process_pid]
+
+        mon_synchronize do
+          if !@clients[process_pid]
+            @clients[process_pid] = {}
+
+            if process_pid != @ancestor_pid
+              @clients.delete(@ancestor_pid)
+            end
+          end
+
+          @clients[process_pid]
+        end
+      end
+
+      def primaries_down
+        process_pid = Process.pid
+        return @primaries_down[process_pid] if @primaries_down[process_pid]
+
+        mon_synchronize do
+          if !@primaries_down[process_pid]
+            @primaries_down[process_pid] = @primaries_down[@ancestor_pid] || {}
+
+            if process_pid != @ancestor_pid
+              @primaries_down.delete(@ancestor_pid).each do |id, options|
+                verify_primary(options)
+              end
+            end
+          end
+
+          @primaries_down[process_pid]
+        end
+      end
+
+      def disconnect_clients(options)
+        key = options[:id]
+
+        mon_synchronize do
+          if clients[key]
+            clients[key].dup.each do |c|
+              c.disconnect
+            end
+          end
+        end
+      end
+
+      def logger
+        RailsFailover::Redis.logger
+      end
+    end
+  end
+end