rails_failover 0.6.1 → 0.6.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: dd1e8f697f988b96e0c149599da43cd69946dd19b06cbe8be89727f6d0775776
-  data.tar.gz: 1119bda19483587f9dc3a7c3a41017b30d66f266342c7e818d1fe9e16a14c997
+  metadata.gz: 152daf08d747b0015a4dddd514ab1478c7fe56e7cfde31900ddbcc7132a32aa8
+  data.tar.gz: 3d64f9b3bad32c9d941e6cd9439e91eab1f4ccc9f3277cd392821eb67d85d9fa
 SHA512:
-  metadata.gz: 4a1f151de2cb49d0728d01de7e62b569f80a025e21401b083a5662e8f967b29c1a6df8b283f02da047b5ce39c190363768b892144d6c28e9dffc2aad2c69d26f
-  data.tar.gz: 9b48b8054f3ae13ec473b7a0869bdbb09ff7f05c6149e9602d1cd2c131383b0be3701ee607aeac2d5cd013e5a59f187a009799e948c4e9bcb7598763ef8932d6
+  metadata.gz: 296bef2bacd7b7ae81f6d707faaaec9ff8fe0cbcd252fd3151e35fe3465d16841687e8c8ad151c1b7ea39f914fe7b95da3f5ced9677e5b2678af87f04bb22d73
+  data.tar.gz: c6780fbf53a99acf790bd425d1856c8b3075ccb83433e577189ccdd674837d41f4b845294f0a822886e2141e37960c21ecde350a7100a0a59242ee714fd162e8

data/CHANGELOG.md

@@ -6,6 +6,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+## [0.6.2] - 2020-11-19
+
+- FIX: Use concurrent-ruby maps to simplify concurrency logic. Resolves a number of possible concurrency issues
+
 ## [0.6.1] - 2020-11-19
 
 - FIX: Recover correctly if both the primary and replica go offline

data/Gemfile.lock

@@ -1,8 +1,9 @@
 PATH
   remote: .
   specs:
-    rails_failover (0.6.1)
+    rails_failover (0.6.2)
       activerecord (~> 6.0)
+      concurrent-ruby
       railties (~> 6.0)
 
 GEM

data/lib/rails_failover/active_record/handler.rb

@@ -1,6 +1,7 @@
 # frozen_string_literal: true
 require 'singleton'
 require 'monitor'
+require 'concurrent'
 
 module RailsFailover
   module ActiveRecord
@@ -11,8 +12,7 @@ module RailsFailover
       VERIFY_FREQUENCY_BUFFER_PRECENT = 20
 
       def initialize
-        @primaries_down = {}
-        @ancestor_pid = Process.pid
+        @primaries_down = Concurrent::Map.new
 
         super() # Monitor#initialize
       end
@@ -22,18 +22,28 @@ module RailsFailover
 
         mon_synchronize do
           return if @thread&.alive?
-
           logger.warn "Failover for ActiveRecord has been initiated"
+          @thread = Thread.new { loop_until_all_up }
+        end
+      end
+
+      def primary_down?(handler_key)
+        primaries_down[handler_key]
+      end
+
+      def primaries_down_count
+        primaries_down.size
+      end
+
+      private
 
-          @thread = Thread.new do
-            loop do
-              initiate_fallback_to_primary
+      def loop_until_all_up
+        loop do
+          initiate_fallback_to_primary
 
-              if all_primaries_up
-                logger.warn "Fallback to primary for ActiveRecord has been completed."
-                break
-              end
-            end
+          if all_primaries_up
+            logger.warn "Fallback to primary for ActiveRecord has been completed."
+            break
           end
         end
       end
@@ -71,37 +81,17 @@ module RailsFailover
         end
       end
 
-      def primary_down?(handler_key)
-        primaries_down[handler_key]
-      end
-
-      def primaries_down_count
-        mon_synchronize do
-          primaries_down.count
-        end
-      end
-
-      private
-
       def all_primaries_up
-        mon_synchronize do
-          primaries_down.empty?
-        end
+        primaries_down.empty?
       end
 
       def primary_down(handler_key)
-        already_down = false
-        mon_synchronize do
-          already_down = !!primaries_down[handler_key]
-          primaries_down[handler_key] = true
-        end
+        already_down = primaries_down.put_if_absent(handler_key, true)
         RailsFailover::ActiveRecord.on_failover_callback!(handler_key) if !already_down
       end
 
       def primary_up(handler_key)
-        already_up = mon_synchronize do
-          !primaries_down.delete(handler_key)
-        end
+        already_up = !primaries_down.delete(handler_key)
         RailsFailover::ActiveRecord.on_fallback_callback!(handler_key) if !already_up
       end
 
@@ -110,24 +100,17 @@ module RailsFailover
       end
 
       def primaries_down
-        process_pid = Process.pid
-        return @primaries_down[process_pid] if @primaries_down[process_pid]
-
-        mon_synchronize do
-          if !@primaries_down[process_pid]
-            @primaries_down[process_pid] = @primaries_down[@ancestor_pid] || {}
-
-            if process_pid != @ancestor_pid
-              @primaries_down.delete(@ancestor_pid)
-
-              @primaries_down[process_pid].each_key do |handler_key|
-                verify_primary(handler_key)
-              end
-            end
-          end
+        ancestor_pids = nil
+        value = @primaries_down.compute_if_absent(Process.pid) do
+          ancestor_pids = @primaries_down.keys
+          @primaries_down.values.first || Concurrent::Map.new
+        end
 
-          @primaries_down[process_pid]
+        ancestor_pids&.each do |pid|
+          @primaries_down.delete(pid)&.each_key { |key| verify_primary(key) }
         end
+
+        value
       end
 
       def logger

data/lib/rails_failover/redis/handler.rb

@@ -2,6 +2,7 @@
 
 require 'monitor'
 require 'singleton'
+require 'concurrent'
 
 module RailsFailover
   class Redis
@@ -16,41 +17,51 @@ module RailsFailover
       SOFT_DISCONNECT_POLL_SECONDS = 0.05
 
       def initialize
-        @primaries_down = {}
-        @clients = {}
-        @ancestor_pid = Process.pid
+        @primaries_down = Concurrent::Map.new
+        @clients = Concurrent::Map.new
 
         super() # Monitor#initialize
       end
 
       def verify_primary(options)
+        primary_down(options)
+
         mon_synchronize do
-          primary_down(options)
-          ensure_failover_thread_running
+          return if @thread&.alive?
+          logger&.warn "Failover for Redis has been initiated"
+          @thread = Thread.new { loop_until_all_up }
         end
       end
 
-      def ensure_failover_thread_running
-        return if @thread&.alive?
+      def register_client(client)
+        id = client.options[:id]
+        clients_for_id(id).put_if_absent(client, true)
+      end
 
-        logger&.warn "Failover for Redis has been initiated"
+      def deregister_client(client)
+        id = client.options[:id]
+        clients_for_id(id).delete(client)
+      end
 
-        @thread = Thread.new do
-          loop do
-            ensure_primary_clients_disconnected
-            try_fallback_to_primary
+      def primary_down?(options)
+        primaries_down[options[:id]]
+      end
 
-            if all_primaries_up
-              logger&.warn "Fallback to primary for Redis has been completed."
-              break
-            end
-          end
-        end
+      def primaries_down_count
+        primaries_down.size
       end
 
-      def ensure_primary_clients_disconnected
-        mon_synchronize { primaries_down.dup }.each do |key, options|
-          disconnect_clients(options, RailsFailover::Redis::PRIMARY)
+      private
+
+      def loop_until_all_up
+        loop do
+          ensure_primary_clients_disconnected
+          try_fallback_to_primary
+
+          if all_primaries_up
+            logger&.warn "Fallback to primary for Redis has been completed."
+            break
+          end
         end
       end
 
@@ -60,7 +71,7 @@ module RailsFailover
 
         active_primaries_keys = {}
 
-        mon_synchronize { primaries_down.dup }.each do |key, options|
+        primaries_down.each do |key, options|
           info = nil
           options = options.dup
 
@@ -87,103 +98,58 @@ module RailsFailover
         end
       end
 
-      def register_client(client)
-        key = client.options[:id]
-
-        mon_synchronize do
-          clients[key] ||= []
-          clients[key] << client
-        end
-      end
-
-      def deregister_client(client)
-        key = client.options[:id]
-
-        mon_synchronize do
-          if clients[key]
-            clients[key].delete(client)
-
-            if clients[key].empty?
-              clients.delete(key)
-            end
-          end
-        end
-      end
-
-      def primary_down?(options)
-        mon_synchronize do
-          primaries_down[options[:id]]
-        end
-      end
-
-      def primaries_down_count
-        mon_synchronize do
-          primaries_down.count
-        end
-      end
-
-      private
-
       def all_primaries_up
-        mon_synchronize { primaries_down.empty? }
+        primaries_down.empty?
       end
 
       def primary_up(options)
-        already_up = mon_synchronize do
-          !primaries_down.delete(options[:id])
-        end
+        already_up = !primaries_down.delete(options[:id])
         RailsFailover::Redis.on_fallback_callback!(options[:id]) if !already_up
       end
 
       def primary_down(options)
-        already_down = false
-        mon_synchronize do
-          already_down = !!primaries_down[options[:id]]
-          primaries_down[options[:id]] = options.dup
-        end
+        already_down = primaries_down.put_if_absent(options[:id], options.dup)
         RailsFailover::Redis.on_failover_callback!(options[:id]) if !already_down
       end
 
-      def clients
-        process_pid = Process.pid
-        return @clients[process_pid] if @clients[process_pid]
-
-        mon_synchronize do
-          if !@clients[process_pid]
-            @clients[process_pid] = {}
-
-            if process_pid != @ancestor_pid
-              @clients.delete(@ancestor_pid)
-            end
-          end
+      def primaries_down
+        ancestor_pids = nil
+        value = @primaries_down.compute_if_absent(Process.pid) do
+          ancestor_pids = @primaries_down.keys
+          @primaries_down.values.first || Concurrent::Map.new
+        end
 
-          @clients[process_pid]
+        ancestor_pids&.each do |pid|
+          @primaries_down.delete(pid)&.each { |id, options| verify_primary(options) }
         end
+
+        value
       end
 
-      def primaries_down
-        process_pid = Process.pid
-        return @primaries_down[process_pid] if @primaries_down[process_pid]
+      def clients_for_id(id)
+        clients.compute_if_absent(id) { Concurrent::Map.new }
+      end
 
-        mon_synchronize do
-          if !@primaries_down[process_pid]
-            @primaries_down[process_pid] = @primaries_down[@ancestor_pid] || {}
-
-            if process_pid != @ancestor_pid
-              @primaries_down.delete(@ancestor_pid)&.each do |id, options|
-                verify_primary(options)
-              end
-            end
-          end
+      def clients
+        ancestor_pids = nil
+        clients_for_pid = @clients.compute_if_absent(Process.pid) do
+          ancestor_pids = @clients.keys
+          Concurrent::Map.new
+        end
+        ancestor_pids&.each { |k| @clients.delete(k) }
+        clients_for_pid
+      end
 
-          @primaries_down[process_pid]
+      def ensure_primary_clients_disconnected
+        primaries_down.each do |key, options|
+          disconnect_clients(options, RailsFailover::Redis::PRIMARY)
         end
       end
 
       def disconnect_clients(options, role)
-        key = options[:id]
+        id = options[:id]
 
-        matched_clients = mon_synchronize { clients[key].dup }
+        matched_clients = clients_for_id(id)&.keys
           &.filter { |c| c.connection.rails_failover_role == role }
           &.to_set
 

data/lib/rails_failover/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module RailsFailover
-  VERSION = "0.6.1"
+  VERSION = "0.6.2"
 end

data/rails_failover.gemspec

@@ -24,4 +24,5 @@ Gem::Specification.new do |spec|
   ["activerecord", "railties"].each do |gem_name|
     spec.add_dependency gem_name, "~> 6.0"
   end
+  spec.add_dependency "concurrent-ruby"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: rails_failover
 version: !ruby/object:Gem::Version
-  version: 0.6.1
+  version: 0.6.2
 platform: ruby
 authors:
 - Alan Tan
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2020-11-20 00:00:00.000000000 Z
+date: 2020-11-25 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activerecord
@@ -38,6 +38,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '6.0'
+- !ruby/object:Gem::Dependency
+  name: concurrent-ruby
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 description: 
 email:
 - tgx@discourse.org