rails_failover 0.5.8 → 0.6.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 1d65e714f38f6431486cd9c2ae8d86e02226f0b1292dca145f056aba3888b05f
-  data.tar.gz: 1c3c4b620ebee9d56ec76523d927149e47cfbc870546483042fe4731e1e7f310
+  metadata.gz: 1586a787b6a0c7573a295515804a0342496cc1f8f9effe1d2e90d34004427dea
+  data.tar.gz: e1358cf8426ea5f6df3f8b039809d5062b18413f6950ecdea20d0379ae68bf5c
 SHA512:
-  metadata.gz: 92751dce5ea5a2d1dcb905f01c9c33678e1cd76999e8d0ee1d59610fd3bcad6f5938a92013a8e2b11f7667a0c50d6b7ac5c8081d113a12f58a429d986e53353a
-  data.tar.gz: b72c80bc12e2852e091d5ad267674cb9238ec7aaa63e3b67304945d0ccc00f55c3f2594a1655d3235ee8b69b362bef00c5793eda5a339fe774a0583f829600ca
+  metadata.gz: d54c7fddc889d91a5f69505b04af146630a13038d3d4d1af05f3e19d083c9ff05fed3ff7e42c0ebe7f576c8e91d6880bdff8466b88990fb27488579a63de7df9
+  data.tar.gz: 88a4cc4c0c7848a0d1eef1ac1bb50d380df2c7bcc4a209fe7ea7733080c63fc929f4d5480e48c945713c2b5083efa967825b909da1ecf52bf414c5b9160d822d

data/CHANGELOG.md

@@ -6,6 +6,34 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+## [0.6.3] - 2020-12-07
+
+- FIX: Handle clients which are connecting during fallback
+
+## [0.6.2] - 2020-11-19
+
+- FIX: Use concurrent-ruby maps to simplify concurrency logic. Resolves a number of possible concurrency issues
+
+## [0.6.1] - 2020-11-19
+
+- FIX: Recover correctly if both the primary and replica go offline
+
+  Previously, a replica failing would cause it to be added to the 'primaries_down' list. The fallback handler would then continuously try and fallback the replica to itself, looping forever, and meaning that fallback to primary would never happen.
+
+## [0.6.0] - 2020-11-09
+- FEATURE: Run failover/fallback callbacks once for each backend
+
+  Previously the failover callback would only fire when the first backend failed, and the fallback callback would only fire when the last backend recovered. Now both failover and fallback callbacks will be triggered for each backend. The key for each backend is also passed to the callbacks for consumption by consuming applications.
+
+- FEATURE: Add primaries_down_count function to failover handlers
+
+  This is intended for consumption by monitoring systems (e.g. the Discourse prometheus exporter) 
+
+## [0.5.9] - 2020-11-06
+- FIX: Ignore errors from the redis socket shutdown call
+
+  This can fail with various i/o errors, but in all cases we want the thread to continue closing the connection with the error, and all the other connections.
+
 ## [0.5.8] - 2020-11-05
 
 - FIX: Handle concurrency issues during redis disconnection (#10)

data/Gemfile.lock

@@ -1,8 +1,9 @@
 PATH
   remote: .
   specs:
-    rails_failover (0.5.8)
+    rails_failover (0.6.3)
       activerecord (~> 6.0)
+      concurrent-ruby
       railties (~> 6.0)
 
 GEM
@@ -38,7 +39,7 @@ GEM
     concurrent-ruby (1.1.6)
     crass (1.0.6)
     diff-lcs (1.3)
-    erubi (1.9.0)
+    erubi (1.10.0)
     i18n (1.8.2)
       concurrent-ruby (~> 1.0)
     loofah (2.7.0)

data/lib/rails_failover/active_record.rb

@@ -47,16 +47,20 @@ module RailsFailover
       @on_failover_callback = block
     end
 
-    def self.on_failover_callback
-      @on_failover_callback
+    def self.on_failover_callback!(key)
+      @on_failover_callback&.call(key)
+    rescue => e
+      logger.warn("RailsFailover::ActiveRecord.on_failover failed: #{e.class} #{e.message}\n#{e.backtrace.join("\n")}")
     end
 
     def self.on_fallback(&block)
       @on_fallback_callback = block
     end
 
-    def self.on_fallback_callback
-      @on_fallback_callback
+    def self.on_fallback_callback!(key)
+      @on_fallback_callback&.call(key)
+    rescue => e
+      logger.warn("RailsFailover::ActiveRecord.on_fallback failed: #{e.class} #{e.message}\n#{e.backtrace.join("\n")}")
     end
   end
 end

data/lib/rails_failover/active_record/handler.rb

@@ -1,6 +1,7 @@
 # frozen_string_literal: true
 require 'singleton'
 require 'monitor'
+require 'concurrent'
 
 module RailsFailover
   module ActiveRecord
@@ -11,41 +12,38 @@ module RailsFailover
       VERIFY_FREQUENCY_BUFFER_PRECENT = 20
 
       def initialize
-        @primaries_down = {}
-        @ancestor_pid = Process.pid
+        @primaries_down = Concurrent::Map.new
 
         super() # Monitor#initialize
       end
 
       def verify_primary(handler_key)
+        primary_down(handler_key)
+
         mon_synchronize do
-          primary_down(handler_key)
           return if @thread&.alive?
-
           logger.warn "Failover for ActiveRecord has been initiated"
+          @thread = Thread.new { loop_until_all_up }
+        end
+      end
 
-          begin
-            RailsFailover::ActiveRecord.on_failover_callback&.call
-          rescue => e
-            logger.warn("RailsFailover::ActiveRecord.on_failover_callback failed: #{e.class} #{e.message}\n#{e.backtrace.join("\n")}")
-          end
+      def primary_down?(handler_key)
+        primaries_down[handler_key]
+      end
 
-          @thread = Thread.new do
-            loop do
-              initiate_fallback_to_primary
+      def primaries_down_count
+        primaries_down.size
+      end
 
-              if all_primaries_up
-                logger.warn "Fallback to primary for ActiveRecord has been completed."
+      private
 
-                begin
-                  RailsFailover::ActiveRecord.on_fallback_callback&.call
-                rescue => e
-                  logger.warn("RailsFailover::ActiveRecord.on_fallback_callback failed: #{e.class} #{e.message}\n#{e.backtrace.join("\n")}")
-                end
+      def loop_until_all_up
+        loop do
+          initiate_fallback_to_primary
 
-                break
-              end
-            end
+          if all_primaries_up
+            logger.warn "Fallback to primary for ActiveRecord has been completed."
+            break
           end
         end
       end
@@ -83,28 +81,18 @@ module RailsFailover
         end
       end
 
-      def primary_down?(handler_key)
-        primaries_down[handler_key]
-      end
-
-      private
-
       def all_primaries_up
-        mon_synchronize do
-          primaries_down.empty?
-        end
+        primaries_down.empty?
       end
 
       def primary_down(handler_key)
-        mon_synchronize do
-          primaries_down[handler_key] = true
-        end
+        already_down = primaries_down.put_if_absent(handler_key, true)
+        RailsFailover::ActiveRecord.on_failover_callback!(handler_key) if !already_down
       end
 
       def primary_up(handler_key)
-        mon_synchronize do
-          primaries_down.delete(handler_key)
-        end
+        already_up = !primaries_down.delete(handler_key)
+        RailsFailover::ActiveRecord.on_fallback_callback!(handler_key) if !already_up
       end
 
       def spec_name
@@ -112,24 +100,17 @@ module RailsFailover
       end
 
       def primaries_down
-        process_pid = Process.pid
-        return @primaries_down[process_pid] if @primaries_down[process_pid]
-
-        mon_synchronize do
-          if !@primaries_down[process_pid]
-            @primaries_down[process_pid] = @primaries_down[@ancestor_pid] || {}
-
-            if process_pid != @ancestor_pid
-              @primaries_down.delete(@ancestor_pid)
-
-              @primaries_down[process_pid].each_key do |handler_key|
-                verify_primary(handler_key)
-              end
-            end
-          end
+        ancestor_pids = nil
+        value = @primaries_down.compute_if_absent(Process.pid) do
+          ancestor_pids = @primaries_down.keys
+          @primaries_down.values.first || Concurrent::Map.new
+        end
 
-          @primaries_down[process_pid]
+        ancestor_pids&.each do |pid|
+          @primaries_down.delete(pid)&.each_key { |key| verify_primary(key) }
         end
+
+        value
       end
 
       def logger

data/lib/rails_failover/redis.rb

@@ -40,16 +40,20 @@ module RailsFailover
       @on_failover_callback = block
     end
 
-    def self.on_failover_callback
-      @on_failover_callback
+    def self.on_failover_callback!(key)
+      @on_failover_callback&.call(key)
+    rescue => e
+      logger.warn("RailsFailover::Redis.on_failover failed: #{e.class} #{e.message}\n#{e.backtrace.join("\n")}")
     end
 
     def self.on_fallback(&block)
       @on_fallback_callback = block
     end
 
-    def self.on_fallback_callback
-      @on_fallback_callback
+    def self.on_fallback_callback!(key)
+      @on_fallback_callback&.call(key)
+    rescue => e
+      logger.warn("RailsFailover::Redis.on_fallback failed: #{e.class} #{e.message}\n#{e.backtrace.join("\n")}")
     end
 
     # For testing

data/lib/rails_failover/redis/connector.rb

@@ -26,7 +26,7 @@ module RailsFailover
                  Errno::ETIMEDOUT,
                  Errno::EINVAL => e
 
-            Handler.instance.verify_primary(options)
+            Handler.instance.verify_primary(options) if !is_failover_replica
             raise e
           end
 
@@ -55,6 +55,10 @@ module RailsFailover
 
       def check(client)
         Handler.instance.register_client(client)
+        expected_role = Handler.instance.primary_down?(@options) ? REPLICA : PRIMARY
+        if client.connection.rails_failover_role != expected_role
+          raise ::Redis::CannotConnectError, "Opened with unexpected failover role"
+        end
       end
 
       def on_disconnect(client)

data/lib/rails_failover/redis/handler.rb

@@ -2,6 +2,7 @@
 
 require 'monitor'
 require 'singleton'
+require 'concurrent'
 
 module RailsFailover
   class Redis
@@ -16,53 +17,51 @@ module RailsFailover
       SOFT_DISCONNECT_POLL_SECONDS = 0.05
 
       def initialize
-        @primaries_down = {}
-        @clients = {}
-        @ancestor_pid = Process.pid
+        @primaries_down = Concurrent::Map.new
+        @clients = Concurrent::Map.new
 
         super() # Monitor#initialize
       end
 
       def verify_primary(options)
+        primary_down(options)
+
         mon_synchronize do
-          primary_down(options)
-          ensure_failover_thread_running
+          return if @thread&.alive?
+          logger&.warn "Failover for Redis has been initiated"
+          @thread = Thread.new { loop_until_all_up }
         end
       end
 
-      def ensure_failover_thread_running
-        return if @thread&.alive?
+      def register_client(client)
+        id = client.options[:id]
+        clients_for_id(id).put_if_absent(client, true)
+      end
 
-        logger&.warn "Failover for Redis has been initiated"
+      def deregister_client(client)
+        id = client.options[:id]
+        clients_for_id(id).delete(client)
+      end
 
-        begin
-          RailsFailover::Redis.on_failover_callback&.call
-        rescue => e
-          logger&.warn("RailsFailover::Redis.on_failover_callback failed: #{e.class} #{e.message}\n#{e.backtrace.join("\n")}")
-        end
+      def primary_down?(options)
+        primaries_down[options[:id]]
+      end
 
-        @thread = Thread.new do
-          loop do
-            ensure_primary_clients_disconnected
-            try_fallback_to_primary
-
-            if all_primaries_up
-              logger&.warn "Fallback to primary for Redis has been completed."
-
-              begin
-                RailsFailover::Redis.on_fallback_callback&.call
-              rescue => e
-                logger&.warn("RailsFailover::Redis.on_fallback_callback failed: #{e.class} #{e.message}\n#{e.backtrace.join("\n")}")
-              end
-              break
-            end
-          end
-        end
+      def primaries_down_count
+        primaries_down.size
       end
 
-      def ensure_primary_clients_disconnected
-        mon_synchronize { primaries_down.dup }.each do |key, options|
-          disconnect_clients(options, RailsFailover::Redis::PRIMARY)
+      private
+
+      def loop_until_all_up
+        loop do
+          ensure_primary_clients_disconnected
+          try_fallback_to_primary
+
+          if all_primaries_up
+            logger&.warn "Fallback to primary for Redis has been completed."
+            break
+          end
         end
       end
 
@@ -72,7 +71,7 @@ module RailsFailover
 
         active_primaries_keys = {}
 
-        mon_synchronize { primaries_down.dup }.each do |key, options|
+        primaries_down.each do |key, options|
           info = nil
           options = options.dup
 
@@ -99,93 +98,58 @@ module RailsFailover
         end
       end
 
-      def register_client(client)
-        key = client.options[:id]
-
-        mon_synchronize do
-          clients[key] ||= []
-          clients[key] << client
-        end
+      def all_primaries_up
+        primaries_down.empty?
       end
 
-      def deregister_client(client)
-        key = client.options[:id]
-
-        mon_synchronize do
-          if clients[key]
-            clients[key].delete(client)
-
-            if clients[key].empty?
-              clients.delete(key)
-            end
-          end
-        end
+      def primary_up(options)
+        already_up = !primaries_down.delete(options[:id])
+        RailsFailover::Redis.on_fallback_callback!(options[:id]) if !already_up
       end
 
-      def primary_down?(options)
-        mon_synchronize do
-          primaries_down[options[:id]]
-        end
+      def primary_down(options)
+        already_down = primaries_down.put_if_absent(options[:id], options.dup)
+        RailsFailover::Redis.on_failover_callback!(options[:id]) if !already_down
       end
 
-      private
-
-      def all_primaries_up
-        mon_synchronize { primaries_down.empty? }
-      end
+      def primaries_down
+        ancestor_pids = nil
+        value = @primaries_down.compute_if_absent(Process.pid) do
+          ancestor_pids = @primaries_down.keys
+          @primaries_down.values.first || Concurrent::Map.new
+        end
 
-      def primary_up(options)
-        mon_synchronize do
-          primaries_down.delete(options[:id])
+        ancestor_pids&.each do |pid|
+          @primaries_down.delete(pid)&.each { |id, options| verify_primary(options) }
         end
+
+        value
       end
 
-      def primary_down(options)
-        mon_synchronize do
-          primaries_down[options[:id]] = options.dup
-        end
+      def clients_for_id(id)
+        clients.compute_if_absent(id) { Concurrent::Map.new }
       end
 
       def clients
-        process_pid = Process.pid
-        return @clients[process_pid] if @clients[process_pid]
-
-        mon_synchronize do
-          if !@clients[process_pid]
-            @clients[process_pid] = {}
-
-            if process_pid != @ancestor_pid
-              @clients.delete(@ancestor_pid)
-            end
-          end
-
-          @clients[process_pid]
+        ancestor_pids = nil
+        clients_for_pid = @clients.compute_if_absent(Process.pid) do
+          ancestor_pids = @clients.keys
+          Concurrent::Map.new
         end
+        ancestor_pids&.each { |k| @clients.delete(k) }
+        clients_for_pid
       end
 
-      def primaries_down
-        process_pid = Process.pid
-        return @primaries_down[process_pid] if @primaries_down[process_pid]
-
-        mon_synchronize do
-          if !@primaries_down[process_pid]
-            @primaries_down[process_pid] = @primaries_down[@ancestor_pid] || {}
-
-            if process_pid != @ancestor_pid
-              @primaries_down.delete(@ancestor_pid)&.each do |id, options|
-                verify_primary(options)
-              end
-            end
-          end
-
-          @primaries_down[process_pid]
+      def ensure_primary_clients_disconnected
+        primaries_down.each do |key, options|
+          disconnect_clients(options, RailsFailover::Redis::PRIMARY)
         end
       end
 
       def disconnect_clients(options, role)
-        key = options[:id]
+        id = options[:id]
 
-        matched_clients = mon_synchronize { clients[key].dup }
+        matched_clients = clients_for_id(id)&.keys
           &.filter { |c| c.connection.rails_failover_role == role }
           &.to_set
 
@@ -208,7 +172,11 @@ module RailsFailover
         has_lock = redis.mon_try_enter
 
         if !has_lock
-          client.connection.shutdown_socket
+          begin
+            client.connection.shutdown_socket
+          rescue => e
+            logger&.warn "Redis shutdown_socket for (#{role}) failed with #{e.class} '#{e.message}'"
+          end
 
           waiting_since = Process.clock_gettime(Process::CLOCK_MONOTONIC)
           loop do # Keep trying

data/lib/rails_failover/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module RailsFailover
-  VERSION = "0.5.8"
+  VERSION = "0.6.3"
 end

data/rails_failover.gemspec

@@ -24,4 +24,5 @@ Gem::Specification.new do |spec|
   ["activerecord", "railties"].each do |gem_name|
     spec.add_dependency gem_name, "~> 6.0"
   end
+  spec.add_dependency "concurrent-ruby"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: rails_failover
 version: !ruby/object:Gem::Version
-  version: 0.5.8
+  version: 0.6.3
 platform: ruby
 authors:
 - Alan Tan
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2020-11-05 00:00:00.000000000 Z
+date: 2020-12-07 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activerecord
@@ -38,6 +38,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '6.0'
+- !ruby/object:Gem::Dependency
+  name: concurrent-ruby
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 description: 
 email:
 - tgx@discourse.org