rails_failover 0.5.7 → 0.5.8

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 6711544c6ae90ab17b509fc3cb97e7b5cc4fd5579daee706dfcc9abb03141a22
-  data.tar.gz: 4bea899af7859b72f88a5e9bd185f72fbca2b968b4a8ed0d73dbc6e41be5acce
+  metadata.gz: 1d65e714f38f6431486cd9c2ae8d86e02226f0b1292dca145f056aba3888b05f
+  data.tar.gz: 1c3c4b620ebee9d56ec76523d927149e47cfbc870546483042fe4731e1e7f310
 SHA512:
-  metadata.gz: 9d58e8aeec4c77b22e8a44d80667b8af553b10323d826fa16da493ab71326e3f0fb4c8f8b4f00fa89f2e53a357642b6c84d138590561c2d389a03de83cac8510
-  data.tar.gz: a6de657f916e8ced227f21c06ab36f168ca8c17adfde474e80526d9a1ab145ab2b00500c93775b22bd6efa35e604fb0c8f9d9b91256caf567a80cb9599c9e270
+  metadata.gz: 92751dce5ea5a2d1dcb905f01c9c33678e1cd76999e8d0ee1d59610fd3bcad6f5938a92013a8e2b11f7667a0c50d6b7ac5c8081d113a12f58a429d986e53353a
+  data.tar.gz: b72c80bc12e2852e091d5ad267674cb9238ec7aaa63e3b67304945d0ccc00f55c3f2594a1655d3235ee8b69b362bef00c5793eda5a339fe774a0583f829600ca

data/.github/workflows/ci.yml

@@ -0,0 +1,81 @@
+name: CI
+
+on:
+  pull_request:
+  push:
+    branches:
+      - master
+    tags:
+      - v*
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+
+    env:
+      BUILD_TYPE: ${{ matrix.build_types }}
+
+    strategy:
+      fail-fast: false
+      matrix:
+        ruby:
+          - 2.6
+          - 2.7
+        build_types: ["LINT", "REDIS", "ACTIVERECORD"]
+        exclude:
+          - ruby: 2.6
+            build_types: "LINT"
+
+    steps:
+      - uses: actions/checkout@v1
+
+      - name: Setup ruby
+        uses: actions/setup-ruby@v1
+        with:
+          ruby-version: ${{ matrix.ruby }}
+          architecture: 'x64'
+
+      - name: Setup bundler
+        run: gem install bundler
+
+      - name: Setup gems
+        run: bundle install
+
+      - name: Rubocop
+        run: bundle exec rubocop
+        if: env.BUILD_TYPE == 'LINT'
+
+      - name: Setup redis
+        run: sudo apt-get install redis-server
+        if: env.BUILD_TYPE == 'REDIS'
+
+      - name: Redis specs
+        run: bin/rspec redis
+        if: env.BUILD_TYPE == 'REDIS'
+
+      - name: Setup test app gems
+        run: cd spec/support/dummy_app && bundle install
+        if: env.BUILD_TYPE == 'ACTIVERECORD'
+
+      - name: Setup postgres
+        run: |
+          make setup_pg
+          make start_pg
+        if: env.BUILD_TYPE == 'ACTIVERECORD'
+
+      - name: ActiveRecord specs
+        run: bin/rspec active_record
+        if: env.BUILD_TYPE == 'ACTIVERECORD'
+
+  publish:
+    if: contains(github.ref, 'refs/tags/v')
+    needs: build
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v2
+
+      - name: Release Gem
+        uses: CvX/publish-rubygems-action@master
+        env:
+          RUBYGEMS_API_KEY: ${{secrets.RUBYGEMS_API_KEY}}

data/CHANGELOG.md

@@ -6,6 +6,15 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+## [0.5.8] - 2020-11-05
+
+- FIX: Handle concurrency issues during redis disconnection (#10)
+
+  This handles concurrency issues which can happen during redis failover/fallback:
+  - Previously, 'subscribed' redis clients were skipped during the disconnect process. This is resolved by directly accessing the original_client from the ::Redis instance
+  - Trying to acquire the mutex on a subscribed redis client is impossible, so the close operation would never complete. Now we send the shutdown() signal to the thread, then allow up to 1 second for the mutex to be released before we close the socket
+  - Failover is almost always triggered inside a redis client mutex. Failover then has its own mutex, within which we attempted to acquire mutexes for all redis clients. This logic causes a deadlock when multiple clients failover simultaneously. Now, all disconnection is performed by the Redis::Handler failover thread, outside of any other mutexes. To make this safe, the primary/replica state is stored in the connection driver, and disconnect_clients is updated to specifically target primary/replica connections.
+
 ## [0.5.7] - 2020-09-16
 
 - FIX: Avoid disconnecting Redis connections abruptly.

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    rails_failover (0.5.7)
+    rails_failover (0.5.8)
       activerecord (~> 6.0)
       railties (~> 6.0)
 

data/lib/rails_failover/redis.rb

@@ -13,6 +13,9 @@ require_relative 'redis/connector'
 
 module RailsFailover
   class Redis
+    PRIMARY = :primary
+    REPLICA = :replica
+
     def self.logger=(logger)
       @logger = logger
     end

data/lib/rails_failover/redis/connector.rb

@@ -10,7 +10,11 @@ module RailsFailover
 
         options[:driver] = Class.new(options[:driver]) do
           def self.connect(options)
-            super(options)
+            is_failover_replica = (options[:host] == options[:replica_host]) &&
+                                  (options[:port] == options[:replica_port])
+            super(options).tap do |conn|
+              conn.rails_failover_role = is_failover_replica ? REPLICA : PRIMARY
+            end
           rescue ::Redis::TimeoutError,
                  SocketError,
                  Errno::EADDRNOTAVAIL,
@@ -25,6 +29,13 @@ module RailsFailover
             Handler.instance.verify_primary(options)
             raise e
           end
+
+          attr_accessor :rails_failover_role
+
+          def shutdown_socket
+            @sock&.shutdown
+          rescue Errno::ENOTCONN
+          end
         end
 
         options[:original_driver] = orignal_driver
@@ -54,8 +65,8 @@ module RailsFailover
 
       def replica_options(options)
         opts = options.dup
-        opts[:host] = opts.delete(:replica_host)
-        opts[:port] = opts.delete(:replica_port)
+        opts[:host] = opts[:replica_host]
+        opts[:port] = opts[:replica_port]
         opts
       end
     end

data/lib/rails_failover/redis/handler.rb

@@ -12,6 +12,8 @@ module RailsFailover
       PRIMARY_ROLE_STATUS = "role:master"
       PRIMARY_LOADED_STATUS = "loading:0"
       VERIFY_FREQUENCY_BUFFER_PRECENT = 20
+      SOFT_DISCONNECT_TIMEOUT_SECONDS = 1
+      SOFT_DISCONNECT_POLL_SECONDS = 0.05
 
       def initialize
         @primaries_down = {}
@@ -24,40 +26,47 @@ module RailsFailover
       def verify_primary(options)
         mon_synchronize do
           primary_down(options)
-          disconnect_clients(options)
-
-          return if @thread&.alive?
+          ensure_failover_thread_running
+        end
+      end
 
-          logger&.warn "Failover for Redis has been initiated"
+      def ensure_failover_thread_running
+        return if @thread&.alive?
 
-          begin
-            RailsFailover::Redis.on_failover_callback&.call
-          rescue => e
-            logger&.warn("RailsFailover::Redis.on_failover_callback failed: #{e.class} #{e.message}\n#{e.backtrace.join("\n")}")
-          end
+        logger&.warn "Failover for Redis has been initiated"
 
-          @thread = Thread.new do
-            loop do
-              thread = Thread.new { initiate_fallback_to_primary }
-              thread.join
+        begin
+          RailsFailover::Redis.on_failover_callback&.call
+        rescue => e
+          logger&.warn("RailsFailover::Redis.on_failover_callback failed: #{e.class} #{e.message}\n#{e.backtrace.join("\n")}")
+        end
 
-              if all_primaries_up
-                logger&.warn "Fallback to primary for Redis has been completed."
+        @thread = Thread.new do
+          loop do
+            ensure_primary_clients_disconnected
+            try_fallback_to_primary
 
-                begin
-                  RailsFailover::Redis.on_fallback_callback&.call
-                rescue => e
-                  logger&.warn("RailsFailover::Redis.on_fallback_callback failed: #{e.class} #{e.message}\n#{e.backtrace.join("\n")}")
-                end
+            if all_primaries_up
+              logger&.warn "Fallback to primary for Redis has been completed."
 
-                break
+              begin
+                RailsFailover::Redis.on_fallback_callback&.call
+              rescue => e
+                logger&.warn("RailsFailover::Redis.on_fallback_callback failed: #{e.class} #{e.message}\n#{e.backtrace.join("\n")}")
               end
+              break
             end
           end
         end
       end
 
-      def initiate_fallback_to_primary
+      def ensure_primary_clients_disconnected
+        mon_synchronize { primaries_down.dup }.each do |key, options|
+          disconnect_clients(options, RailsFailover::Redis::PRIMARY)
+        end
+      end
+
+      def try_fallback_to_primary
         frequency = RailsFailover::Redis.verify_primary_frequency_seconds
         sleep(frequency * ((rand(VERIFY_FREQUENCY_BUFFER_PRECENT) + 100) / 100.0))
 
@@ -86,7 +95,7 @@ module RailsFailover
 
         active_primaries_keys.each do |key, options|
           primary_up(options)
-          disconnect_clients(options)
+          disconnect_clients(options, RailsFailover::Redis::REPLICA)
         end
       end
 
@@ -173,23 +182,48 @@ module RailsFailover
         end
       end
 
-      def disconnect_clients(options)
+      def disconnect_clients(options, role)
         key = options[:id]
 
-        mon_synchronize do
-          if to_disconnect = clients[key].dup
-            # Don't disconnect connections abruptly since it may lead to unexepcted
-            # errors. Is there a better way to do this without having to monkey patch
-            # the redis-rb gem heavily?
-            ObjectSpace.each_object(::Redis).each do |redis|
-              to_disconnect.each do |c|
-                if redis._client == c
-                  redis.synchronize { |_client| _client.disconnect }
-                end
-              end
-            end
+        matched_clients = mon_synchronize { clients[key].dup }
+          &.filter { |c| c.connection.rails_failover_role == role }
+          &.to_set
+
+        return if matched_clients.nil? || matched_clients.empty?
+
+        # This is not ideal, but the mutex we need is contained
+        # in the ::Redis instance, not the Redis::Client
+        ObjectSpace.each_object(::Redis) do |redis|
+          # When subscribed, Redis#_client is not a Redis::Client
+          # Instance variable is the only reliable way
+          client = redis.instance_variable_get(:@original_client)
+          next if !matched_clients.include?(client)
+          soft_disconnect(redis, client, role)
+        end
+      end
+
+      # Calling .disconnect can cause a running subscribe() to block forever
+      # Therefore try to acquire the lock
+      def soft_disconnect(redis, client, role)
+        has_lock = redis.mon_try_enter
+
+        if !has_lock
+          client.connection.shutdown_socket
+
+          waiting_since = Process.clock_gettime(Process::CLOCK_MONOTONIC)
+          loop do # Keep trying
+            break if has_lock = redis.mon_try_enter
+            break if !client.connection.connected? # Disconnected by other thread
+            break if client.connection.rails_failover_role != role # Reconnected by other thread
+            time_now = Process.clock_gettime(Process::CLOCK_MONOTONIC)
+            break if time_now > waiting_since + SOFT_DISCONNECT_TIMEOUT_SECONDS
+            sleep SOFT_DISCONNECT_POLL_SECONDS
           end
         end
+
+        client.disconnect if client.connection&.rails_failover_role == role
+      ensure
+        redis.mon_exit if has_lock
       end
 
       def logger

data/lib/rails_failover/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module RailsFailover
-  VERSION = "0.5.7"
+  VERSION = "0.5.8"
 end

data/makefile

@@ -18,4 +18,4 @@ stop_dummy_rails_server:
 	@kill -TERM $(shell cat spec/support/dummy_app/tmp/pids/unicorn.pid)
 
 teardown_dummy_rails_server:
-	@cd spec/support/dummy_app && DISABLE_DATABASE_ENVIRONMENT_CHECK=1 RAILS_ENV=production $(BUNDLER_BIN) exec rails db:drop
+	@cd spec/support/dummy_app && (! (bundle check > /dev/null 2>&1) || DISABLE_DATABASE_ENVIRONMENT_CHECK=1 RAILS_ENV=production $(BUNDLER_BIN) exec rails db:drop)

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: rails_failover
 version: !ruby/object:Gem::Version
-  version: 0.5.7
+  version: 0.5.8
 platform: ruby
 authors:
 - Alan Tan
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2020-09-16 00:00:00.000000000 Z
+date: 2020-11-05 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activerecord
@@ -45,10 +45,10 @@ executables: []
 extensions: []
 extra_rdoc_files: []
 files:
+- ".github/workflows/ci.yml"
 - ".gitignore"
 - ".rspec"
 - ".rubocop.yml"
-- ".travis.yml"
 - CHANGELOG.md
 - CODE_OF_CONDUCT.md
 - Gemfile
@@ -92,7 +92,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.1.2
+rubygems_version: 3.0.3
 signing_key: 
 specification_version: 4
 summary: Failover for ActiveRecord and Redis

data/.travis.yml

@@ -1,6 +0,0 @@
----
-language: ruby
-cache: bundler
-rvm:
-  - 2.6.2
-before_install: gem install bundler -v 2.1.4