rails_failover 0.5.7 → 0.6.2

Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: 6711544c6ae90ab17b509fc3cb97e7b5cc4fd5579daee706dfcc9abb03141a22
4
- data.tar.gz: 4bea899af7859b72f88a5e9bd185f72fbca2b968b4a8ed0d73dbc6e41be5acce
3
+ metadata.gz: 152daf08d747b0015a4dddd514ab1478c7fe56e7cfde31900ddbcc7132a32aa8
4
+ data.tar.gz: 3d64f9b3bad32c9d941e6cd9439e91eab1f4ccc9f3277cd392821eb67d85d9fa
5
5
  SHA512:
6
- metadata.gz: 9d58e8aeec4c77b22e8a44d80667b8af553b10323d826fa16da493ab71326e3f0fb4c8f8b4f00fa89f2e53a357642b6c84d138590561c2d389a03de83cac8510
7
- data.tar.gz: a6de657f916e8ced227f21c06ab36f168ca8c17adfde474e80526d9a1ab145ab2b00500c93775b22bd6efa35e604fb0c8f9d9b91256caf567a80cb9599c9e270
6
+ metadata.gz: 296bef2bacd7b7ae81f6d707faaaec9ff8fe0cbcd252fd3151e35fe3465d16841687e8c8ad151c1b7ea39f914fe7b95da3f5ced9677e5b2678af87f04bb22d73
7
+ data.tar.gz: c6780fbf53a99acf790bd425d1856c8b3075ccb83433e577189ccdd674837d41f4b845294f0a822886e2141e37960c21ecde350a7100a0a59242ee714fd162e8
@@ -0,0 +1,81 @@
1
+ name: CI
2
+
3
+ on:
4
+ pull_request:
5
+ push:
6
+ branches:
7
+ - master
8
+ tags:
9
+ - v*
10
+
11
+ jobs:
12
+ build:
13
+ runs-on: ubuntu-latest
14
+
15
+ env:
16
+ BUILD_TYPE: ${{ matrix.build_types }}
17
+
18
+ strategy:
19
+ fail-fast: false
20
+ matrix:
21
+ ruby:
22
+ - 2.6
23
+ - 2.7
24
+ build_types: ["LINT", "REDIS", "ACTIVERECORD"]
25
+ exclude:
26
+ - ruby: 2.6
27
+ build_types: "LINT"
28
+
29
+ steps:
30
+ - uses: actions/checkout@v1
31
+
32
+ - name: Setup ruby
33
+ uses: actions/setup-ruby@v1
34
+ with:
35
+ ruby-version: ${{ matrix.ruby }}
36
+ architecture: 'x64'
37
+
38
+ - name: Setup bundler
39
+ run: gem install bundler
40
+
41
+ - name: Setup gems
42
+ run: bundle install
43
+
44
+ - name: Rubocop
45
+ run: bundle exec rubocop
46
+ if: env.BUILD_TYPE == 'LINT'
47
+
48
+ - name: Setup redis
49
+ run: sudo apt-get install redis-server
50
+ if: env.BUILD_TYPE == 'REDIS'
51
+
52
+ - name: Redis specs
53
+ run: bin/rspec redis
54
+ if: env.BUILD_TYPE == 'REDIS'
55
+
56
+ - name: Setup test app gems
57
+ run: cd spec/support/dummy_app && bundle install
58
+ if: env.BUILD_TYPE == 'ACTIVERECORD'
59
+
60
+ - name: Setup postgres
61
+ run: |
62
+ make setup_pg
63
+ make start_pg
64
+ if: env.BUILD_TYPE == 'ACTIVERECORD'
65
+
66
+ - name: ActiveRecord specs
67
+ run: bin/rspec active_record
68
+ if: env.BUILD_TYPE == 'ACTIVERECORD'
69
+
70
+ publish:
71
+ if: contains(github.ref, 'refs/tags/v')
72
+ needs: build
73
+ runs-on: ubuntu-latest
74
+
75
+ steps:
76
+ - uses: actions/checkout@v2
77
+
78
+ - name: Release Gem
79
+ uses: CvX/publish-rubygems-action@master
80
+ env:
81
+ RUBYGEMS_API_KEY: ${{secrets.RUBYGEMS_API_KEY}}
@@ -6,6 +6,39 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
6
6
 
7
7
  ## [Unreleased]
8
8
 
9
+ ## [0.6.2] - 2020-11-19
10
+
11
+ - FIX: Use concurrent-ruby maps to simplify concurrency logic. Resolves a number of possible concurrency issues
12
+
13
+ ## [0.6.1] - 2020-11-19
14
+
15
+ - FIX: Recover correctly if both the primary and replica go offline
16
+
17
+ Previously, a replica failing would cause it to be added to the 'primaries_down' list. The fallback handler would then continuously try and fallback the replica to itself, looping forever, and meaning that fallback to primary would never happen.
18
+
19
+ ## [0.6.0] - 2020-11-09
20
+ - FEATURE: Run failover/fallback callbacks once for each backend
21
+
22
+ Previously the failover callback would only fire when the first backend failed, and the fallback callback would only fire when the last backend recovered. Now both failover and fallback callbacks will be triggered for each backend. The key for each backend is also passed to the callbacks for consumption by consuming applications.
23
+
24
+ - FEATURE: Add primaries_down_count function to failover handlers
25
+
26
+ This is intended for consumption by monitoring systems (e.g. the Discourse prometheus exporter)
27
+
28
+ ## [0.5.9] - 2020-11-06
29
+ - FIX: Ignore errors from the redis socket shutdown call
30
+
31
+ This can fail with various i/o errors, but in all cases we want the thread to continue closing the connection with the error, and all the other connections.
32
+
33
+ ## [0.5.8] - 2020-11-05
34
+
35
+ - FIX: Handle concurrency issues during redis disconnection (#10)
36
+
37
+ This handles concurrency issues which can happen during redis failover/fallback:
38
+ - Previously, 'subscribed' redis clients were skipped during the disconnect process. This is resolved by directly accessing the original_client from the ::Redis instance
39
+ - Trying to acquire the mutex on a subscribed redis client is impossible, so the close operation would never complete. Now we send the shutdown() signal to the thread, then allow up to 1 second for the mutex to be released before we close the socket
40
+ - Failover is almost always triggered inside a redis client mutex. Failover then has its own mutex, within which we attempted to acquire mutexes for all redis clients. This logic causes a deadlock when multiple clients failover simultaneously. Now, all disconnection is performed by the Redis::Handler failover thread, outside of any other mutexes. To make this safe, the primary/replica state is stored in the connection driver, and disconnect_clients is updated to specifically target primary/replica connections.
41
+
9
42
  ## [0.5.7] - 2020-09-16
10
43
 
11
44
  - FIX: Avoid disconnecting Redis connections abruptly.
@@ -1,8 +1,9 @@
1
1
  PATH
2
2
  remote: .
3
3
  specs:
4
- rails_failover (0.5.7)
4
+ rails_failover (0.6.2)
5
5
  activerecord (~> 6.0)
6
+ concurrent-ruby
6
7
  railties (~> 6.0)
7
8
 
8
9
  GEM
@@ -38,7 +39,7 @@ GEM
38
39
  concurrent-ruby (1.1.6)
39
40
  crass (1.0.6)
40
41
  diff-lcs (1.3)
41
- erubi (1.9.0)
42
+ erubi (1.10.0)
42
43
  i18n (1.8.2)
43
44
  concurrent-ruby (~> 1.0)
44
45
  loofah (2.7.0)
@@ -47,16 +47,20 @@ module RailsFailover
47
47
  @on_failover_callback = block
48
48
  end
49
49
 
50
- def self.on_failover_callback
51
- @on_failover_callback
50
+ def self.on_failover_callback!(key)
51
+ @on_failover_callback&.call(key)
52
+ rescue => e
53
+ logger.warn("RailsFailover::ActiveRecord.on_failover failed: #{e.class} #{e.message}\n#{e.backtrace.join("\n")}")
52
54
  end
53
55
 
54
56
  def self.on_fallback(&block)
55
57
  @on_fallback_callback = block
56
58
  end
57
59
 
58
- def self.on_fallback_callback
59
- @on_fallback_callback
60
+ def self.on_fallback_callback!(key)
61
+ @on_fallback_callback&.call(key)
62
+ rescue => e
63
+ logger.warn("RailsFailover::ActiveRecord.on_fallback failed: #{e.class} #{e.message}\n#{e.backtrace.join("\n")}")
60
64
  end
61
65
  end
62
66
  end
@@ -1,6 +1,7 @@
1
1
  # frozen_string_literal: true
2
2
  require 'singleton'
3
3
  require 'monitor'
4
+ require 'concurrent'
4
5
 
5
6
  module RailsFailover
6
7
  module ActiveRecord
@@ -11,41 +12,38 @@ module RailsFailover
11
12
  VERIFY_FREQUENCY_BUFFER_PRECENT = 20
12
13
 
13
14
  def initialize
14
- @primaries_down = {}
15
- @ancestor_pid = Process.pid
15
+ @primaries_down = Concurrent::Map.new
16
16
 
17
17
  super() # Monitor#initialize
18
18
  end
19
19
 
20
20
  def verify_primary(handler_key)
21
+ primary_down(handler_key)
22
+
21
23
  mon_synchronize do
22
- primary_down(handler_key)
23
24
  return if @thread&.alive?
24
-
25
25
  logger.warn "Failover for ActiveRecord has been initiated"
26
+ @thread = Thread.new { loop_until_all_up }
27
+ end
28
+ end
26
29
 
27
- begin
28
- RailsFailover::ActiveRecord.on_failover_callback&.call
29
- rescue => e
30
- logger.warn("RailsFailover::ActiveRecord.on_failover_callback failed: #{e.class} #{e.message}\n#{e.backtrace.join("\n")}")
31
- end
30
+ def primary_down?(handler_key)
31
+ primaries_down[handler_key]
32
+ end
32
33
 
33
- @thread = Thread.new do
34
- loop do
35
- initiate_fallback_to_primary
34
+ def primaries_down_count
35
+ primaries_down.size
36
+ end
36
37
 
37
- if all_primaries_up
38
- logger.warn "Fallback to primary for ActiveRecord has been completed."
38
+ private
39
39
 
40
- begin
41
- RailsFailover::ActiveRecord.on_fallback_callback&.call
42
- rescue => e
43
- logger.warn("RailsFailover::ActiveRecord.on_fallback_callback failed: #{e.class} #{e.message}\n#{e.backtrace.join("\n")}")
44
- end
40
+ def loop_until_all_up
41
+ loop do
42
+ initiate_fallback_to_primary
45
43
 
46
- break
47
- end
48
- end
44
+ if all_primaries_up
45
+ logger.warn "Fallback to primary for ActiveRecord has been completed."
46
+ break
49
47
  end
50
48
  end
51
49
  end
@@ -83,28 +81,18 @@ module RailsFailover
83
81
  end
84
82
  end
85
83
 
86
- def primary_down?(handler_key)
87
- primaries_down[handler_key]
88
- end
89
-
90
- private
91
-
92
84
  def all_primaries_up
93
- mon_synchronize do
94
- primaries_down.empty?
95
- end
85
+ primaries_down.empty?
96
86
  end
97
87
 
98
88
  def primary_down(handler_key)
99
- mon_synchronize do
100
- primaries_down[handler_key] = true
101
- end
89
+ already_down = primaries_down.put_if_absent(handler_key, true)
90
+ RailsFailover::ActiveRecord.on_failover_callback!(handler_key) if !already_down
102
91
  end
103
92
 
104
93
  def primary_up(handler_key)
105
- mon_synchronize do
106
- primaries_down.delete(handler_key)
107
- end
94
+ already_up = !primaries_down.delete(handler_key)
95
+ RailsFailover::ActiveRecord.on_fallback_callback!(handler_key) if !already_up
108
96
  end
109
97
 
110
98
  def spec_name
@@ -112,24 +100,17 @@ module RailsFailover
112
100
  end
113
101
 
114
102
  def primaries_down
115
- process_pid = Process.pid
116
- return @primaries_down[process_pid] if @primaries_down[process_pid]
117
-
118
- mon_synchronize do
119
- if !@primaries_down[process_pid]
120
- @primaries_down[process_pid] = @primaries_down[@ancestor_pid] || {}
121
-
122
- if process_pid != @ancestor_pid
123
- @primaries_down.delete(@ancestor_pid)
124
-
125
- @primaries_down[process_pid].each_key do |handler_key|
126
- verify_primary(handler_key)
127
- end
128
- end
129
- end
103
+ ancestor_pids = nil
104
+ value = @primaries_down.compute_if_absent(Process.pid) do
105
+ ancestor_pids = @primaries_down.keys
106
+ @primaries_down.values.first || Concurrent::Map.new
107
+ end
130
108
 
131
- @primaries_down[process_pid]
109
+ ancestor_pids&.each do |pid|
110
+ @primaries_down.delete(pid)&.each_key { |key| verify_primary(key) }
132
111
  end
112
+
113
+ value
133
114
  end
134
115
 
135
116
  def logger
@@ -13,6 +13,9 @@ require_relative 'redis/connector'
13
13
 
14
14
  module RailsFailover
15
15
  class Redis
16
+ PRIMARY = :primary
17
+ REPLICA = :replica
18
+
16
19
  def self.logger=(logger)
17
20
  @logger = logger
18
21
  end
@@ -37,16 +40,20 @@ module RailsFailover
37
40
  @on_failover_callback = block
38
41
  end
39
42
 
40
- def self.on_failover_callback
41
- @on_failover_callback
43
+ def self.on_failover_callback!(key)
44
+ @on_failover_callback&.call(key)
45
+ rescue => e
46
+ logger.warn("RailsFailover::Redis.on_failover failed: #{e.class} #{e.message}\n#{e.backtrace.join("\n")}")
42
47
  end
43
48
 
44
49
  def self.on_fallback(&block)
45
50
  @on_fallback_callback = block
46
51
  end
47
52
 
48
- def self.on_fallback_callback
49
- @on_fallback_callback
53
+ def self.on_fallback_callback!(key)
54
+ @on_fallback_callback&.call(key)
55
+ rescue => e
56
+ logger.warn("RailsFailover::Redis.on_fallback failed: #{e.class} #{e.message}\n#{e.backtrace.join("\n")}")
50
57
  end
51
58
 
52
59
  # For testing
@@ -10,7 +10,11 @@ module RailsFailover
10
10
 
11
11
  options[:driver] = Class.new(options[:driver]) do
12
12
  def self.connect(options)
13
- super(options)
13
+ is_failover_replica = (options[:host] == options[:replica_host]) &&
14
+ (options[:port] == options[:replica_port])
15
+ super(options).tap do |conn|
16
+ conn.rails_failover_role = is_failover_replica ? REPLICA : PRIMARY
17
+ end
14
18
  rescue ::Redis::TimeoutError,
15
19
  SocketError,
16
20
  Errno::EADDRNOTAVAIL,
@@ -22,9 +26,16 @@ module RailsFailover
22
26
  Errno::ETIMEDOUT,
23
27
  Errno::EINVAL => e
24
28
 
25
- Handler.instance.verify_primary(options)
29
+ Handler.instance.verify_primary(options) if !is_failover_replica
26
30
  raise e
27
31
  end
32
+
33
+ attr_accessor :rails_failover_role
34
+
35
+ def shutdown_socket
36
+ @sock&.shutdown
37
+ rescue Errno::ENOTCONN
38
+ end
28
39
  end
29
40
 
30
41
  options[:original_driver] = orignal_driver
@@ -54,8 +65,8 @@ module RailsFailover
54
65
 
55
66
  def replica_options(options)
56
67
  opts = options.dup
57
- opts[:host] = opts.delete(:replica_host)
58
- opts[:port] = opts.delete(:replica_port)
68
+ opts[:host] = opts[:replica_host]
69
+ opts[:port] = opts[:replica_port]
59
70
  opts
60
71
  end
61
72
  end
@@ -2,6 +2,7 @@
2
2
 
3
3
  require 'monitor'
4
4
  require 'singleton'
5
+ require 'concurrent'
5
6
 
6
7
  module RailsFailover
7
8
  class Redis
@@ -12,58 +13,65 @@ module RailsFailover
12
13
  PRIMARY_ROLE_STATUS = "role:master"
13
14
  PRIMARY_LOADED_STATUS = "loading:0"
14
15
  VERIFY_FREQUENCY_BUFFER_PRECENT = 20
16
+ SOFT_DISCONNECT_TIMEOUT_SECONDS = 1
17
+ SOFT_DISCONNECT_POLL_SECONDS = 0.05
15
18
 
16
19
  def initialize
17
- @primaries_down = {}
18
- @clients = {}
19
- @ancestor_pid = Process.pid
20
+ @primaries_down = Concurrent::Map.new
21
+ @clients = Concurrent::Map.new
20
22
 
21
23
  super() # Monitor#initialize
22
24
  end
23
25
 
24
26
  def verify_primary(options)
25
- mon_synchronize do
26
- primary_down(options)
27
- disconnect_clients(options)
27
+ primary_down(options)
28
28
 
29
+ mon_synchronize do
29
30
  return if @thread&.alive?
30
-
31
31
  logger&.warn "Failover for Redis has been initiated"
32
+ @thread = Thread.new { loop_until_all_up }
33
+ end
34
+ end
32
35
 
33
- begin
34
- RailsFailover::Redis.on_failover_callback&.call
35
- rescue => e
36
- logger&.warn("RailsFailover::Redis.on_failover_callback failed: #{e.class} #{e.message}\n#{e.backtrace.join("\n")}")
37
- end
36
+ def register_client(client)
37
+ id = client.options[:id]
38
+ clients_for_id(id).put_if_absent(client, true)
39
+ end
40
+
41
+ def deregister_client(client)
42
+ id = client.options[:id]
43
+ clients_for_id(id).delete(client)
44
+ end
45
+
46
+ def primary_down?(options)
47
+ primaries_down[options[:id]]
48
+ end
38
49
 
39
- @thread = Thread.new do
40
- loop do
41
- thread = Thread.new { initiate_fallback_to_primary }
42
- thread.join
50
+ def primaries_down_count
51
+ primaries_down.size
52
+ end
43
53
 
44
- if all_primaries_up
45
- logger&.warn "Fallback to primary for Redis has been completed."
54
+ private
46
55
 
47
- begin
48
- RailsFailover::Redis.on_fallback_callback&.call
49
- rescue => e
50
- logger&.warn("RailsFailover::Redis.on_fallback_callback failed: #{e.class} #{e.message}\n#{e.backtrace.join("\n")}")
51
- end
56
+ def loop_until_all_up
57
+ loop do
58
+ ensure_primary_clients_disconnected
59
+ try_fallback_to_primary
52
60
 
53
- break
54
- end
55
- end
61
+ if all_primaries_up
62
+ logger&.warn "Fallback to primary for Redis has been completed."
63
+ break
56
64
  end
57
65
  end
58
66
  end
59
67
 
60
- def initiate_fallback_to_primary
68
+ def try_fallback_to_primary
61
69
  frequency = RailsFailover::Redis.verify_primary_frequency_seconds
62
70
  sleep(frequency * ((rand(VERIFY_FREQUENCY_BUFFER_PRECENT) + 100) / 100.0))
63
71
 
64
72
  active_primaries_keys = {}
65
73
 
66
- mon_synchronize { primaries_down.dup }.each do |key, options|
74
+ primaries_down.each do |key, options|
67
75
  info = nil
68
76
  options = options.dup
69
77
 
@@ -86,110 +94,104 @@ module RailsFailover
86
94
 
87
95
  active_primaries_keys.each do |key, options|
88
96
  primary_up(options)
89
- disconnect_clients(options)
97
+ disconnect_clients(options, RailsFailover::Redis::REPLICA)
90
98
  end
91
99
  end
92
100
 
93
- def register_client(client)
94
- key = client.options[:id]
95
-
96
- mon_synchronize do
97
- clients[key] ||= []
98
- clients[key] << client
99
- end
101
+ def all_primaries_up
102
+ primaries_down.empty?
100
103
  end
101
104
 
102
- def deregister_client(client)
103
- key = client.options[:id]
105
+ def primary_up(options)
106
+ already_up = !primaries_down.delete(options[:id])
107
+ RailsFailover::Redis.on_fallback_callback!(options[:id]) if !already_up
108
+ end
104
109
 
105
- mon_synchronize do
106
- if clients[key]
107
- clients[key].delete(client)
110
+ def primary_down(options)
111
+ already_down = primaries_down.put_if_absent(options[:id], options.dup)
112
+ RailsFailover::Redis.on_failover_callback!(options[:id]) if !already_down
113
+ end
108
114
 
109
- if clients[key].empty?
110
- clients.delete(key)
111
- end
112
- end
115
+ def primaries_down
116
+ ancestor_pids = nil
117
+ value = @primaries_down.compute_if_absent(Process.pid) do
118
+ ancestor_pids = @primaries_down.keys
119
+ @primaries_down.values.first || Concurrent::Map.new
113
120
  end
114
- end
115
121
 
116
- def primary_down?(options)
117
- mon_synchronize do
118
- primaries_down[options[:id]]
122
+ ancestor_pids&.each do |pid|
123
+ @primaries_down.delete(pid)&.each { |id, options| verify_primary(options) }
119
124
  end
120
- end
121
125
 
122
- private
126
+ value
127
+ end
123
128
 
124
- def all_primaries_up
125
- mon_synchronize { primaries_down.empty? }
129
+ def clients_for_id(id)
130
+ clients.compute_if_absent(id) { Concurrent::Map.new }
126
131
  end
127
132
 
128
- def primary_up(options)
129
- mon_synchronize do
130
- primaries_down.delete(options[:id])
133
+ def clients
134
+ ancestor_pids = nil
135
+ clients_for_pid = @clients.compute_if_absent(Process.pid) do
136
+ ancestor_pids = @clients.keys
137
+ Concurrent::Map.new
131
138
  end
139
+ ancestor_pids&.each { |k| @clients.delete(k) }
140
+ clients_for_pid
132
141
  end
133
142
 
134
- def primary_down(options)
135
- mon_synchronize do
136
- primaries_down[options[:id]] = options.dup
143
+ def ensure_primary_clients_disconnected
144
+ primaries_down.each do |key, options|
145
+ disconnect_clients(options, RailsFailover::Redis::PRIMARY)
137
146
  end
138
147
  end
139
148
 
140
- def clients
141
- process_pid = Process.pid
142
- return @clients[process_pid] if @clients[process_pid]
149
+ def disconnect_clients(options, role)
150
+ id = options[:id]
143
151
 
144
- mon_synchronize do
145
- if !@clients[process_pid]
146
- @clients[process_pid] = {}
152
+ matched_clients = clients_for_id(id)&.keys
153
+ &.filter { |c| c.connection.rails_failover_role == role }
154
+ &.to_set
147
155
 
148
- if process_pid != @ancestor_pid
149
- @clients.delete(@ancestor_pid)
150
- end
151
- end
156
+ return if matched_clients.nil? || matched_clients.empty?
152
157
 
153
- @clients[process_pid]
158
+ # This is not ideal, but the mutex we need is contained
159
+ # in the ::Redis instance, not the Redis::Client
160
+ ObjectSpace.each_object(::Redis) do |redis|
161
+ # When subscribed, Redis#_client is not a Redis::Client
162
+ # Instance variable is the only reliable way
163
+ client = redis.instance_variable_get(:@original_client)
164
+ next if !matched_clients.include?(client)
165
+ soft_disconnect(redis, client, role)
154
166
  end
155
167
  end
156
168
 
157
- def primaries_down
158
- process_pid = Process.pid
159
- return @primaries_down[process_pid] if @primaries_down[process_pid]
169
+ # Calling .disconnect can cause a running subscribe() to block forever
170
+ # Therefore try to acquire the lock
171
+ def soft_disconnect(redis, client, role)
172
+ has_lock = redis.mon_try_enter
160
173
 
161
- mon_synchronize do
162
- if !@primaries_down[process_pid]
163
- @primaries_down[process_pid] = @primaries_down[@ancestor_pid] || {}
164
-
165
- if process_pid != @ancestor_pid
166
- @primaries_down.delete(@ancestor_pid)&.each do |id, options|
167
- verify_primary(options)
168
- end
169
- end
174
+ if !has_lock
175
+ begin
176
+ client.connection.shutdown_socket
177
+ rescue => e
178
+ logger&.warn "Redis shutdown_socket for (#{role}) failed with #{e.class} '#{e.message}'"
170
179
  end
171
180
 
172
- @primaries_down[process_pid]
173
- end
174
- end
175
-
176
- def disconnect_clients(options)
177
- key = options[:id]
178
-
179
- mon_synchronize do
180
- if to_disconnect = clients[key].dup
181
- # Don't disconnect connections abruptly since it may lead to unexepcted
182
- # errors. Is there a better way to do this without having to monkey patch
183
- # the redis-rb gem heavily?
184
- ObjectSpace.each_object(::Redis).each do |redis|
185
- to_disconnect.each do |c|
186
- if redis._client == c
187
- redis.synchronize { |_client| _client.disconnect }
188
- end
189
- end
190
- end
181
+ waiting_since = Process.clock_gettime(Process::CLOCK_MONOTONIC)
182
+ loop do # Keep trying
183
+ break if has_lock = redis.mon_try_enter
184
+ break if !client.connection.connected? # Disconnected by other thread
185
+ break if client.connection.rails_failover_role != role # Reconnected by other thread
186
+ time_now = Process.clock_gettime(Process::CLOCK_MONOTONIC)
187
+ break if time_now > waiting_since + SOFT_DISCONNECT_TIMEOUT_SECONDS
188
+ sleep SOFT_DISCONNECT_POLL_SECONDS
191
189
  end
192
190
  end
191
+
192
+ client.disconnect if client.connection&.rails_failover_role == role
193
+ ensure
194
+ redis.mon_exit if has_lock
193
195
  end
194
196
 
195
197
  def logger
@@ -1,5 +1,5 @@
1
1
  # frozen_string_literal: true
2
2
 
3
3
  module RailsFailover
4
- VERSION = "0.5.7"
4
+ VERSION = "0.6.2"
5
5
  end
data/makefile CHANGED
@@ -18,4 +18,4 @@ stop_dummy_rails_server:
18
18
  @kill -TERM $(shell cat spec/support/dummy_app/tmp/pids/unicorn.pid)
19
19
 
20
20
  teardown_dummy_rails_server:
21
- @cd spec/support/dummy_app && DISABLE_DATABASE_ENVIRONMENT_CHECK=1 RAILS_ENV=production $(BUNDLER_BIN) exec rails db:drop
21
+ @cd spec/support/dummy_app && (! (bundle check > /dev/null 2>&1) || DISABLE_DATABASE_ENVIRONMENT_CHECK=1 RAILS_ENV=production $(BUNDLER_BIN) exec rails db:drop)
@@ -24,4 +24,5 @@ Gem::Specification.new do |spec|
24
24
  ["activerecord", "railties"].each do |gem_name|
25
25
  spec.add_dependency gem_name, "~> 6.0"
26
26
  end
27
+ spec.add_dependency "concurrent-ruby"
27
28
  end
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: rails_failover
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.5.7
4
+ version: 0.6.2
5
5
  platform: ruby
6
6
  authors:
7
7
  - Alan Tan
8
8
  autorequire:
9
9
  bindir: exe
10
10
  cert_chain: []
11
- date: 2020-09-16 00:00:00.000000000 Z
11
+ date: 2020-11-25 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: activerecord
@@ -38,6 +38,20 @@ dependencies:
38
38
  - - "~>"
39
39
  - !ruby/object:Gem::Version
40
40
  version: '6.0'
41
+ - !ruby/object:Gem::Dependency
42
+ name: concurrent-ruby
43
+ requirement: !ruby/object:Gem::Requirement
44
+ requirements:
45
+ - - ">="
46
+ - !ruby/object:Gem::Version
47
+ version: '0'
48
+ type: :runtime
49
+ prerelease: false
50
+ version_requirements: !ruby/object:Gem::Requirement
51
+ requirements:
52
+ - - ">="
53
+ - !ruby/object:Gem::Version
54
+ version: '0'
41
55
  description:
42
56
  email:
43
57
  - tgx@discourse.org
@@ -45,10 +59,10 @@ executables: []
45
59
  extensions: []
46
60
  extra_rdoc_files: []
47
61
  files:
62
+ - ".github/workflows/ci.yml"
48
63
  - ".gitignore"
49
64
  - ".rspec"
50
65
  - ".rubocop.yml"
51
- - ".travis.yml"
52
66
  - CHANGELOG.md
53
67
  - CODE_OF_CONDUCT.md
54
68
  - Gemfile
@@ -92,7 +106,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
92
106
  - !ruby/object:Gem::Version
93
107
  version: '0'
94
108
  requirements: []
95
- rubygems_version: 3.1.2
109
+ rubygems_version: 3.0.3
96
110
  signing_key:
97
111
  specification_version: 4
98
112
  summary: Failover for ActiveRecord and Redis
@@ -1,6 +0,0 @@
1
- ---
2
- language: ruby
3
- cache: bundler
4
- rvm:
5
- - 2.6.2
6
- before_install: gem install bundler -v 2.1.4