rails_failover 0.6.0 → 0.6.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: dea60f6f50e09e45029bc4f73d89eadefd569935d6855c3542f82aa3e1a01b3f
-  data.tar.gz: 5ccd925be0977242e1f139481b59a3feb3e0a4a9dbf19f8cf21064466fcdbcb4
+  metadata.gz: 45afa97c24a3c8f4fac5c74a52eabbf0d06b9f7ad037a76b18bee05875b4f374
+  data.tar.gz: e57ffc9182a9b2f4145c04d4f921bcd79aee3edfec5dedacac2fb6f23d39dc9d
 SHA512:
-  metadata.gz: d39ff2d6eb505dff92151c884bb7ec3aa608d4bd29c79eea2a0e2e965b8eec6ab787f930128e0d3ffcbbfb1eb2cedccf1a8fa30417eff4d3a2cfea40d536d0a7
-  data.tar.gz: 2b36aa85cff837f80b0876ce7b2af3017c935601ec5c5329f2bab26f5ccb395d8dd50111bee7dbf3ad2bb6e15cb22f2aa18f9fff89b3a101689f1d5036371f7e
+  metadata.gz: 0ee4ae4e79e17881a49f77711b5f13f301977d67cd177f8d16296cff93c0aea63b9151f235acfde87afaae14f6215840fbb3efb6024a98b59c14454b88705356
+  data.tar.gz: 48d815e482d5b6b92316431bda654709f3049647292c4a4d9b8a6a075086be2e23b12cceed898d07e10f729bc85217aa395521a49ffd1c4baad63927ed938963

data/CHANGELOG.md

@@ -6,6 +6,28 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+## [0.6.5] - 2020-12-16
+
+- FIX: Catch exceptions that are not intercepted by `ActionDispatch::DebugExceptions`.
+
+## [0.6.4] - 2020-12-09
+
+- FIX: Handle the case when the replica is set equal to the primary
+
+## [0.6.3] - 2020-12-07
+
+- FIX: Handle clients which are connecting during fallback
+
+## [0.6.2] - 2020-11-19
+
+- FIX: Use concurrent-ruby maps to simplify concurrency logic. Resolves a number of possible concurrency issues
+
+## [0.6.1] - 2020-11-19
+
+- FIX: Recover correctly if both the primary and replica go offline
+
+  Previously, a replica failing would cause it to be added to the 'primaries_down' list. The fallback handler would then continuously try and fallback the replica to itself, looping forever, and meaning that fallback to primary would never happen.
+
 ## [0.6.0] - 2020-11-09
 - FEATURE: Run failover/fallback callbacks once for each backend
 
@@ -13,7 +35,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 - FEATURE: Add primaries_down_count function to failover handlers
 
-  This is intended for consumption by monitoring systems (e.g. the Discourse prometheus exporter) 
+  This is intended for consumption by monitoring systems (e.g. the Discourse prometheus exporter)
 
 ## [0.5.9] - 2020-11-06
 - FIX: Ignore errors from the redis socket shutdown call

data/Gemfile.lock

@@ -1,8 +1,9 @@
 PATH
   remote: .
   specs:
-    rails_failover (0.6.0)
+    rails_failover (0.6.5)
       activerecord (~> 6.0)
+      concurrent-ruby
       railties (~> 6.0)
 
 GEM
@@ -38,7 +39,7 @@ GEM
     concurrent-ruby (1.1.6)
     crass (1.0.6)
     diff-lcs (1.3)
-    erubi (1.9.0)
+    erubi (1.10.0)
     i18n (1.8.2)
       concurrent-ruby (~> 1.0)
     loofah (2.7.0)

data/lib/rails_failover/active_record/handler.rb

@@ -1,6 +1,7 @@
 # frozen_string_literal: true
 require 'singleton'
 require 'monitor'
+require 'concurrent'
 
 module RailsFailover
   module ActiveRecord
@@ -11,8 +12,7 @@ module RailsFailover
       VERIFY_FREQUENCY_BUFFER_PRECENT = 20
 
       def initialize
-        @primaries_down = {}
-        @ancestor_pid = Process.pid
+        @primaries_down = Concurrent::Map.new
 
         super() # Monitor#initialize
       end
@@ -22,18 +22,28 @@ module RailsFailover
 
         mon_synchronize do
           return if @thread&.alive?
-
           logger.warn "Failover for ActiveRecord has been initiated"
+          @thread = Thread.new { loop_until_all_up }
+        end
+      end
+
+      def primary_down?(handler_key)
+        primaries_down[handler_key]
+      end
+
+      def primaries_down_count
+        primaries_down.size
+      end
+
+      private
 
-          @thread = Thread.new do
-            loop do
-              initiate_fallback_to_primary
+      def loop_until_all_up
+        loop do
+          initiate_fallback_to_primary
 
-              if all_primaries_up
-                logger.warn "Fallback to primary for ActiveRecord has been completed."
-                break
-              end
-            end
+          if all_primaries_up
+            logger.warn "Fallback to primary for ActiveRecord has been completed."
+            break
           end
         end
       end
@@ -71,37 +81,17 @@ module RailsFailover
         end
       end
 
-      def primary_down?(handler_key)
-        primaries_down[handler_key]
-      end
-
-      def primaries_down_count
-        mon_synchronize do
-          primaries_down.count
-        end
-      end
-
-      private
-
       def all_primaries_up
-        mon_synchronize do
-          primaries_down.empty?
-        end
+        primaries_down.empty?
       end
 
       def primary_down(handler_key)
-        already_down = false
-        mon_synchronize do
-          already_down = !!primaries_down[handler_key]
-          primaries_down[handler_key] = true
-        end
+        already_down = primaries_down.put_if_absent(handler_key, true)
         RailsFailover::ActiveRecord.on_failover_callback!(handler_key) if !already_down
       end
 
       def primary_up(handler_key)
-        already_up = mon_synchronize do
-          !primaries_down.delete(handler_key)
-        end
+        already_up = !primaries_down.delete(handler_key)
         RailsFailover::ActiveRecord.on_fallback_callback!(handler_key) if !already_up
       end
 
@@ -110,24 +100,17 @@ module RailsFailover
       end
 
       def primaries_down
-        process_pid = Process.pid
-        return @primaries_down[process_pid] if @primaries_down[process_pid]
-
-        mon_synchronize do
-          if !@primaries_down[process_pid]
-            @primaries_down[process_pid] = @primaries_down[@ancestor_pid] || {}
-
-            if process_pid != @ancestor_pid
-              @primaries_down.delete(@ancestor_pid)
-
-              @primaries_down[process_pid].each_key do |handler_key|
-                verify_primary(handler_key)
-              end
-            end
-          end
+        ancestor_pids = nil
+        value = @primaries_down.compute_if_absent(Process.pid) do
+          ancestor_pids = @primaries_down.keys
+          @primaries_down.values.first || Concurrent::Map.new
+        end
 
-          @primaries_down[process_pid]
+        ancestor_pids&.each do |pid|
+          @primaries_down.delete(pid)&.each_key { |key| verify_primary(key) }
         end
+
+        value
       end
 
       def logger

data/lib/rails_failover/active_record/middleware.rb

@@ -14,10 +14,17 @@ module RailsFailover
       end
 
       def self.handle(request, exception)
+        verify_primary(
+          exception,
+          request.env[Middleware::WRITING_ROLE_HEADER]
+        )
+      end
+
+      def self.verify_primary(exception, writing_role)
         exception = resolve_cause(exception)
 
         if adapter_errors.any? { |error| exception.is_a?(error) }
-          Handler.instance.verify_primary(request.env[Middleware::WRITING_ROLE_HEADER])
+          Handler.instance.verify_primary(writing_role)
         end
       end
 
@@ -61,6 +68,9 @@ module RailsFailover
           env[WRITING_ROLE_HEADER] = writing_role
           @app.call(env)
         end
+      rescue => e
+        Interceptor.verify_primary(e, writing_role) if writing_role
+        raise e
       end
 
       private

data/lib/rails_failover/redis/connector.rb

@@ -7,13 +7,15 @@ module RailsFailover
     class Connector < ::Redis::Client::Connector
       def initialize(options)
         orignal_driver = options[:driver]
+        options[:primary_host] = options[:host]
+        options[:primary_port] = options[:port]
 
         options[:driver] = Class.new(options[:driver]) do
           def self.connect(options)
-            is_failover_replica = (options[:host] == options[:replica_host]) &&
-                                  (options[:port] == options[:replica_port])
+            is_primary = (options[:host] == options[:primary_host]) &&
+                         (options[:port] == options[:primary_port])
             super(options).tap do |conn|
-              conn.rails_failover_role = is_failover_replica ? REPLICA : PRIMARY
+              conn.rails_failover_role = is_primary ? PRIMARY : REPLICA
             end
           rescue ::Redis::TimeoutError,
                  SocketError,
@@ -26,7 +28,7 @@ module RailsFailover
                  Errno::ETIMEDOUT,
                  Errno::EINVAL => e
 
-            Handler.instance.verify_primary(options)
+            Handler.instance.verify_primary(options) if is_primary
             raise e
           end
 
@@ -55,6 +57,10 @@ module RailsFailover
 
       def check(client)
         Handler.instance.register_client(client)
+        expected_role = Handler.instance.primary_down?(@options) ? REPLICA : PRIMARY
+        if client.connection.rails_failover_role != expected_role
+          raise ::Redis::CannotConnectError, "Opened with unexpected failover role"
+        end
       end
 
       def on_disconnect(client)

data/lib/rails_failover/redis/handler.rb

@@ -2,6 +2,7 @@
 
 require 'monitor'
 require 'singleton'
+require 'concurrent'
 
 module RailsFailover
   class Redis
@@ -16,41 +17,51 @@ module RailsFailover
       SOFT_DISCONNECT_POLL_SECONDS = 0.05
 
       def initialize
-        @primaries_down = {}
-        @clients = {}
-        @ancestor_pid = Process.pid
+        @primaries_down = Concurrent::Map.new
+        @clients = Concurrent::Map.new
 
         super() # Monitor#initialize
       end
 
       def verify_primary(options)
+        primary_down(options)
+
         mon_synchronize do
-          primary_down(options)
-          ensure_failover_thread_running
+          return if @thread&.alive?
+          logger&.warn "Failover for Redis has been initiated"
+          @thread = Thread.new { loop_until_all_up }
         end
       end
 
-      def ensure_failover_thread_running
-        return if @thread&.alive?
+      def register_client(client)
+        id = client.options[:id]
+        clients_for_id(id).put_if_absent(client, true)
+      end
 
-        logger&.warn "Failover for Redis has been initiated"
+      def deregister_client(client)
+        id = client.options[:id]
+        clients_for_id(id).delete(client)
+      end
 
-        @thread = Thread.new do
-          loop do
-            ensure_primary_clients_disconnected
-            try_fallback_to_primary
+      def primary_down?(options)
+        primaries_down[options[:id]]
+      end
 
-            if all_primaries_up
-              logger&.warn "Fallback to primary for Redis has been completed."
-              break
-            end
-          end
-        end
+      def primaries_down_count
+        primaries_down.size
       end
 
-      def ensure_primary_clients_disconnected
-        mon_synchronize { primaries_down.dup }.each do |key, options|
-          disconnect_clients(options, RailsFailover::Redis::PRIMARY)
+      private
+
+      def loop_until_all_up
+        loop do
+          ensure_primary_clients_disconnected
+          try_fallback_to_primary
+
+          if all_primaries_up
+            logger&.warn "Fallback to primary for Redis has been completed."
+            break
+          end
         end
       end
 
@@ -60,7 +71,7 @@ module RailsFailover
 
         active_primaries_keys = {}
 
-        mon_synchronize { primaries_down.dup }.each do |key, options|
+        primaries_down.each do |key, options|
           info = nil
           options = options.dup
 
@@ -87,103 +98,58 @@ module RailsFailover
         end
       end
 
-      def register_client(client)
-        key = client.options[:id]
-
-        mon_synchronize do
-          clients[key] ||= []
-          clients[key] << client
-        end
-      end
-
-      def deregister_client(client)
-        key = client.options[:id]
-
-        mon_synchronize do
-          if clients[key]
-            clients[key].delete(client)
-
-            if clients[key].empty?
-              clients.delete(key)
-            end
-          end
-        end
-      end
-
-      def primary_down?(options)
-        mon_synchronize do
-          primaries_down[options[:id]]
-        end
-      end
-
-      def primaries_down_count
-        mon_synchronize do
-          primaries_down.count
-        end
-      end
-
-      private
-
       def all_primaries_up
-        mon_synchronize { primaries_down.empty? }
+        primaries_down.empty?
       end
 
       def primary_up(options)
-        already_up = mon_synchronize do
-          !primaries_down.delete(options[:id])
-        end
+        already_up = !primaries_down.delete(options[:id])
         RailsFailover::Redis.on_fallback_callback!(options[:id]) if !already_up
       end
 
       def primary_down(options)
-        already_down = false
-        mon_synchronize do
-          already_down = !!primaries_down[options[:id]]
-          primaries_down[options[:id]] = options.dup
-        end
+        already_down = primaries_down.put_if_absent(options[:id], options.dup)
         RailsFailover::Redis.on_failover_callback!(options[:id]) if !already_down
       end
 
-      def clients
-        process_pid = Process.pid
-        return @clients[process_pid] if @clients[process_pid]
-
-        mon_synchronize do
-          if !@clients[process_pid]
-            @clients[process_pid] = {}
-
-            if process_pid != @ancestor_pid
-              @clients.delete(@ancestor_pid)
-            end
-          end
+      def primaries_down
+        ancestor_pids = nil
+        value = @primaries_down.compute_if_absent(Process.pid) do
+          ancestor_pids = @primaries_down.keys
+          @primaries_down.values.first || Concurrent::Map.new
+        end
 
-          @clients[process_pid]
+        ancestor_pids&.each do |pid|
+          @primaries_down.delete(pid)&.each { |id, options| verify_primary(options) }
         end
+
+        value
       end
 
-      def primaries_down
-        process_pid = Process.pid
-        return @primaries_down[process_pid] if @primaries_down[process_pid]
+      def clients_for_id(id)
+        clients.compute_if_absent(id) { Concurrent::Map.new }
+      end
 
-        mon_synchronize do
-          if !@primaries_down[process_pid]
-            @primaries_down[process_pid] = @primaries_down[@ancestor_pid] || {}
-
-            if process_pid != @ancestor_pid
-              @primaries_down.delete(@ancestor_pid)&.each do |id, options|
-                verify_primary(options)
-              end
-            end
-          end
+      def clients
+        ancestor_pids = nil
+        clients_for_pid = @clients.compute_if_absent(Process.pid) do
+          ancestor_pids = @clients.keys
+          Concurrent::Map.new
+        end
+        ancestor_pids&.each { |k| @clients.delete(k) }
+        clients_for_pid
+      end
 
-          @primaries_down[process_pid]
+      def ensure_primary_clients_disconnected
+        primaries_down.each do |key, options|
+          disconnect_clients(options, RailsFailover::Redis::PRIMARY)
         end
       end
 
       def disconnect_clients(options, role)
-        key = options[:id]
+        id = options[:id]
 
-        matched_clients = mon_synchronize { clients[key].dup }
+        matched_clients = clients_for_id(id)&.keys
           &.filter { |c| c.connection.rails_failover_role == role }
           &.to_set
 

data/lib/rails_failover/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module RailsFailover
-  VERSION = "0.6.0"
+  VERSION = "0.6.5"
 end

data/makefile

@@ -9,7 +9,7 @@ test_active_record:
 	@ACTIVE_RECORD=1 bundle exec rspec --tag type:active_record ${RSPEC_PATH}
 
 setup_dummy_rails_server:
-	@cd spec/support/dummy_app && bundle install --quiet --without test --without development && yarn install && RAILS_ENV=production $(BUNDLER_BIN) exec rails db:create db:migrate db:seed
+	@cd spec/support/dummy_app && bundle install --quiet && yarn install && RAILS_ENV=production $(BUNDLER_BIN) exec rails db:create db:migrate db:seed
 
 start_dummy_rails_server:
 	@cd spec/support/dummy_app && BUNDLE_GEMFILE=Gemfile UNICORN_WORKERS=5 SECRET_KEY_BASE=somekey bundle exec unicorn -c config/unicorn.conf.rb -D -E production

data/rails_failover.gemspec

@@ -24,4 +24,5 @@ Gem::Specification.new do |spec|
   ["activerecord", "railties"].each do |gem_name|
     spec.add_dependency gem_name, "~> 6.0"
   end
+  spec.add_dependency "concurrent-ruby"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: rails_failover
 version: !ruby/object:Gem::Version
-  version: 0.6.0
+  version: 0.6.5
 platform: ruby
 authors:
 - Alan Tan
-autorequire: 
+autorequire:
 bindir: exe
 cert_chain: []
-date: 2020-11-09 00:00:00.000000000 Z
+date: 2020-12-16 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activerecord
@@ -38,7 +38,21 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '6.0'
-description: 
+- !ruby/object:Gem::Dependency
+  name: concurrent-ruby
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description:
 email:
 - tgx@discourse.org
 executables: []
@@ -73,11 +87,11 @@ files:
 - postgresql.mk
 - rails_failover.gemspec
 - redis.mk
-homepage: 
+homepage:
 licenses:
 - MIT
 metadata: {}
-post_install_message: 
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -92,8 +106,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.0.3
-signing_key: 
+rubygems_version: 3.1.4
+signing_key:
 specification_version: 4
 summary: Failover for ActiveRecord and Redis
 test_files: []