rails_failover 0.5.7 → 0.6.2
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.github/workflows/ci.yml +81 -0
- data/CHANGELOG.md +33 -0
- data/Gemfile.lock +3 -2
- data/lib/rails_failover/active_record.rb +8 -4
- data/lib/rails_failover/active_record/handler.rb +34 -53
- data/lib/rails_failover/redis.rb +11 -4
- data/lib/rails_failover/redis/connector.rb +15 -4
- data/lib/rails_failover/redis/handler.rb +103 -101
- data/lib/rails_failover/version.rb +1 -1
- data/makefile +1 -1
- data/rails_failover.gemspec +1 -0
- metadata +18 -4
- data/.travis.yml +0 -6
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 152daf08d747b0015a4dddd514ab1478c7fe56e7cfde31900ddbcc7132a32aa8
|
4
|
+
data.tar.gz: 3d64f9b3bad32c9d941e6cd9439e91eab1f4ccc9f3277cd392821eb67d85d9fa
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 296bef2bacd7b7ae81f6d707faaaec9ff8fe0cbcd252fd3151e35fe3465d16841687e8c8ad151c1b7ea39f914fe7b95da3f5ced9677e5b2678af87f04bb22d73
|
7
|
+
data.tar.gz: c6780fbf53a99acf790bd425d1856c8b3075ccb83433e577189ccdd674837d41f4b845294f0a822886e2141e37960c21ecde350a7100a0a59242ee714fd162e8
|
@@ -0,0 +1,81 @@
|
|
1
|
+
name: CI
|
2
|
+
|
3
|
+
on:
|
4
|
+
pull_request:
|
5
|
+
push:
|
6
|
+
branches:
|
7
|
+
- master
|
8
|
+
tags:
|
9
|
+
- v*
|
10
|
+
|
11
|
+
jobs:
|
12
|
+
build:
|
13
|
+
runs-on: ubuntu-latest
|
14
|
+
|
15
|
+
env:
|
16
|
+
BUILD_TYPE: ${{ matrix.build_types }}
|
17
|
+
|
18
|
+
strategy:
|
19
|
+
fail-fast: false
|
20
|
+
matrix:
|
21
|
+
ruby:
|
22
|
+
- 2.6
|
23
|
+
- 2.7
|
24
|
+
build_types: ["LINT", "REDIS", "ACTIVERECORD"]
|
25
|
+
exclude:
|
26
|
+
- ruby: 2.6
|
27
|
+
build_types: "LINT"
|
28
|
+
|
29
|
+
steps:
|
30
|
+
- uses: actions/checkout@v1
|
31
|
+
|
32
|
+
- name: Setup ruby
|
33
|
+
uses: actions/setup-ruby@v1
|
34
|
+
with:
|
35
|
+
ruby-version: ${{ matrix.ruby }}
|
36
|
+
architecture: 'x64'
|
37
|
+
|
38
|
+
- name: Setup bundler
|
39
|
+
run: gem install bundler
|
40
|
+
|
41
|
+
- name: Setup gems
|
42
|
+
run: bundle install
|
43
|
+
|
44
|
+
- name: Rubocop
|
45
|
+
run: bundle exec rubocop
|
46
|
+
if: env.BUILD_TYPE == 'LINT'
|
47
|
+
|
48
|
+
- name: Setup redis
|
49
|
+
run: sudo apt-get install redis-server
|
50
|
+
if: env.BUILD_TYPE == 'REDIS'
|
51
|
+
|
52
|
+
- name: Redis specs
|
53
|
+
run: bin/rspec redis
|
54
|
+
if: env.BUILD_TYPE == 'REDIS'
|
55
|
+
|
56
|
+
- name: Setup test app gems
|
57
|
+
run: cd spec/support/dummy_app && bundle install
|
58
|
+
if: env.BUILD_TYPE == 'ACTIVERECORD'
|
59
|
+
|
60
|
+
- name: Setup postgres
|
61
|
+
run: |
|
62
|
+
make setup_pg
|
63
|
+
make start_pg
|
64
|
+
if: env.BUILD_TYPE == 'ACTIVERECORD'
|
65
|
+
|
66
|
+
- name: ActiveRecord specs
|
67
|
+
run: bin/rspec active_record
|
68
|
+
if: env.BUILD_TYPE == 'ACTIVERECORD'
|
69
|
+
|
70
|
+
publish:
|
71
|
+
if: contains(github.ref, 'refs/tags/v')
|
72
|
+
needs: build
|
73
|
+
runs-on: ubuntu-latest
|
74
|
+
|
75
|
+
steps:
|
76
|
+
- uses: actions/checkout@v2
|
77
|
+
|
78
|
+
- name: Release Gem
|
79
|
+
uses: CvX/publish-rubygems-action@master
|
80
|
+
env:
|
81
|
+
RUBYGEMS_API_KEY: ${{secrets.RUBYGEMS_API_KEY}}
|
data/CHANGELOG.md
CHANGED
@@ -6,6 +6,39 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
|
|
6
6
|
|
7
7
|
## [Unreleased]
|
8
8
|
|
9
|
+
## [0.6.2] - 2020-11-19
|
10
|
+
|
11
|
+
- FIX: Use concurrent-ruby maps to simplify concurrency logic. Resolves a number of possible concurrency issues
|
12
|
+
|
13
|
+
## [0.6.1] - 2020-11-19
|
14
|
+
|
15
|
+
- FIX: Recover correctly if both the primary and replica go offline
|
16
|
+
|
17
|
+
Previously, a replica failing would cause it to be added to the 'primaries_down' list. The fallback handler would then continuously try and fallback the replica to itself, looping forever, and meaning that fallback to primary would never happen.
|
18
|
+
|
19
|
+
## [0.6.0] - 2020-11-09
|
20
|
+
- FEATURE: Run failover/fallback callbacks once for each backend
|
21
|
+
|
22
|
+
Previously the failover callback would only fire when the first backend failed, and the fallback callback would only fire when the last backend recovered. Now both failover and fallback callbacks will be triggered for each backend. The key for each backend is also passed to the callbacks for consumption by consuming applications.
|
23
|
+
|
24
|
+
- FEATURE: Add primaries_down_count function to failover handlers
|
25
|
+
|
26
|
+
This is intended for consumption by monitoring systems (e.g. the Discourse prometheus exporter)
|
27
|
+
|
28
|
+
## [0.5.9] - 2020-11-06
|
29
|
+
- FIX: Ignore errors from the redis socket shutdown call
|
30
|
+
|
31
|
+
This can fail with various i/o errors, but in all cases we want the thread to continue closing the connection with the error, and all the other connections.
|
32
|
+
|
33
|
+
## [0.5.8] - 2020-11-05
|
34
|
+
|
35
|
+
- FIX: Handle concurrency issues during redis disconnection (#10)
|
36
|
+
|
37
|
+
This handles concurrency issues which can happen during redis failover/fallback:
|
38
|
+
- Previously, 'subscribed' redis clients were skipped during the disconnect process. This is resolved by directly accessing the original_client from the ::Redis instance
|
39
|
+
- Trying to acquire the mutex on a subscribed redis client is impossible, so the close operation would never complete. Now we send the shutdown() signal to the thread, then allow up to 1 second for the mutex to be released before we close the socket
|
40
|
+
- Failover is almost always triggered inside a redis client mutex. Failover then has its own mutex, within which we attempted to acquire mutexes for all redis clients. This logic causes a deadlock when multiple clients failover simultaneously. Now, all disconnection is performed by the Redis::Handler failover thread, outside of any other mutexes. To make this safe, the primary/replica state is stored in the connection driver, and disconnect_clients is updated to specifically target primary/replica connections.
|
41
|
+
|
9
42
|
## [0.5.7] - 2020-09-16
|
10
43
|
|
11
44
|
- FIX: Avoid disconnecting Redis connections abruptly.
|
data/Gemfile.lock
CHANGED
@@ -1,8 +1,9 @@
|
|
1
1
|
PATH
|
2
2
|
remote: .
|
3
3
|
specs:
|
4
|
-
rails_failover (0.
|
4
|
+
rails_failover (0.6.2)
|
5
5
|
activerecord (~> 6.0)
|
6
|
+
concurrent-ruby
|
6
7
|
railties (~> 6.0)
|
7
8
|
|
8
9
|
GEM
|
@@ -38,7 +39,7 @@ GEM
|
|
38
39
|
concurrent-ruby (1.1.6)
|
39
40
|
crass (1.0.6)
|
40
41
|
diff-lcs (1.3)
|
41
|
-
erubi (1.
|
42
|
+
erubi (1.10.0)
|
42
43
|
i18n (1.8.2)
|
43
44
|
concurrent-ruby (~> 1.0)
|
44
45
|
loofah (2.7.0)
|
@@ -47,16 +47,20 @@ module RailsFailover
|
|
47
47
|
@on_failover_callback = block
|
48
48
|
end
|
49
49
|
|
50
|
-
def self.on_failover_callback
|
51
|
-
@on_failover_callback
|
50
|
+
def self.on_failover_callback!(key)
|
51
|
+
@on_failover_callback&.call(key)
|
52
|
+
rescue => e
|
53
|
+
logger.warn("RailsFailover::ActiveRecord.on_failover failed: #{e.class} #{e.message}\n#{e.backtrace.join("\n")}")
|
52
54
|
end
|
53
55
|
|
54
56
|
def self.on_fallback(&block)
|
55
57
|
@on_fallback_callback = block
|
56
58
|
end
|
57
59
|
|
58
|
-
def self.on_fallback_callback
|
59
|
-
@on_fallback_callback
|
60
|
+
def self.on_fallback_callback!(key)
|
61
|
+
@on_fallback_callback&.call(key)
|
62
|
+
rescue => e
|
63
|
+
logger.warn("RailsFailover::ActiveRecord.on_fallback failed: #{e.class} #{e.message}\n#{e.backtrace.join("\n")}")
|
60
64
|
end
|
61
65
|
end
|
62
66
|
end
|
@@ -1,6 +1,7 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
require 'singleton'
|
3
3
|
require 'monitor'
|
4
|
+
require 'concurrent'
|
4
5
|
|
5
6
|
module RailsFailover
|
6
7
|
module ActiveRecord
|
@@ -11,41 +12,38 @@ module RailsFailover
|
|
11
12
|
VERIFY_FREQUENCY_BUFFER_PRECENT = 20
|
12
13
|
|
13
14
|
def initialize
|
14
|
-
@primaries_down =
|
15
|
-
@ancestor_pid = Process.pid
|
15
|
+
@primaries_down = Concurrent::Map.new
|
16
16
|
|
17
17
|
super() # Monitor#initialize
|
18
18
|
end
|
19
19
|
|
20
20
|
def verify_primary(handler_key)
|
21
|
+
primary_down(handler_key)
|
22
|
+
|
21
23
|
mon_synchronize do
|
22
|
-
primary_down(handler_key)
|
23
24
|
return if @thread&.alive?
|
24
|
-
|
25
25
|
logger.warn "Failover for ActiveRecord has been initiated"
|
26
|
+
@thread = Thread.new { loop_until_all_up }
|
27
|
+
end
|
28
|
+
end
|
26
29
|
|
27
|
-
|
28
|
-
|
29
|
-
|
30
|
-
logger.warn("RailsFailover::ActiveRecord.on_failover_callback failed: #{e.class} #{e.message}\n#{e.backtrace.join("\n")}")
|
31
|
-
end
|
30
|
+
def primary_down?(handler_key)
|
31
|
+
primaries_down[handler_key]
|
32
|
+
end
|
32
33
|
|
33
|
-
|
34
|
-
|
35
|
-
|
34
|
+
def primaries_down_count
|
35
|
+
primaries_down.size
|
36
|
+
end
|
36
37
|
|
37
|
-
|
38
|
-
logger.warn "Fallback to primary for ActiveRecord has been completed."
|
38
|
+
private
|
39
39
|
|
40
|
-
|
41
|
-
|
42
|
-
|
43
|
-
logger.warn("RailsFailover::ActiveRecord.on_fallback_callback failed: #{e.class} #{e.message}\n#{e.backtrace.join("\n")}")
|
44
|
-
end
|
40
|
+
def loop_until_all_up
|
41
|
+
loop do
|
42
|
+
initiate_fallback_to_primary
|
45
43
|
|
46
|
-
|
47
|
-
|
48
|
-
|
44
|
+
if all_primaries_up
|
45
|
+
logger.warn "Fallback to primary for ActiveRecord has been completed."
|
46
|
+
break
|
49
47
|
end
|
50
48
|
end
|
51
49
|
end
|
@@ -83,28 +81,18 @@ module RailsFailover
|
|
83
81
|
end
|
84
82
|
end
|
85
83
|
|
86
|
-
def primary_down?(handler_key)
|
87
|
-
primaries_down[handler_key]
|
88
|
-
end
|
89
|
-
|
90
|
-
private
|
91
|
-
|
92
84
|
def all_primaries_up
|
93
|
-
|
94
|
-
primaries_down.empty?
|
95
|
-
end
|
85
|
+
primaries_down.empty?
|
96
86
|
end
|
97
87
|
|
98
88
|
def primary_down(handler_key)
|
99
|
-
|
100
|
-
|
101
|
-
end
|
89
|
+
already_down = primaries_down.put_if_absent(handler_key, true)
|
90
|
+
RailsFailover::ActiveRecord.on_failover_callback!(handler_key) if !already_down
|
102
91
|
end
|
103
92
|
|
104
93
|
def primary_up(handler_key)
|
105
|
-
|
106
|
-
|
107
|
-
end
|
94
|
+
already_up = !primaries_down.delete(handler_key)
|
95
|
+
RailsFailover::ActiveRecord.on_fallback_callback!(handler_key) if !already_up
|
108
96
|
end
|
109
97
|
|
110
98
|
def spec_name
|
@@ -112,24 +100,17 @@ module RailsFailover
|
|
112
100
|
end
|
113
101
|
|
114
102
|
def primaries_down
|
115
|
-
|
116
|
-
|
117
|
-
|
118
|
-
|
119
|
-
|
120
|
-
@primaries_down[process_pid] = @primaries_down[@ancestor_pid] || {}
|
121
|
-
|
122
|
-
if process_pid != @ancestor_pid
|
123
|
-
@primaries_down.delete(@ancestor_pid)
|
124
|
-
|
125
|
-
@primaries_down[process_pid].each_key do |handler_key|
|
126
|
-
verify_primary(handler_key)
|
127
|
-
end
|
128
|
-
end
|
129
|
-
end
|
103
|
+
ancestor_pids = nil
|
104
|
+
value = @primaries_down.compute_if_absent(Process.pid) do
|
105
|
+
ancestor_pids = @primaries_down.keys
|
106
|
+
@primaries_down.values.first || Concurrent::Map.new
|
107
|
+
end
|
130
108
|
|
131
|
-
|
109
|
+
ancestor_pids&.each do |pid|
|
110
|
+
@primaries_down.delete(pid)&.each_key { |key| verify_primary(key) }
|
132
111
|
end
|
112
|
+
|
113
|
+
value
|
133
114
|
end
|
134
115
|
|
135
116
|
def logger
|
data/lib/rails_failover/redis.rb
CHANGED
@@ -13,6 +13,9 @@ require_relative 'redis/connector'
|
|
13
13
|
|
14
14
|
module RailsFailover
|
15
15
|
class Redis
|
16
|
+
PRIMARY = :primary
|
17
|
+
REPLICA = :replica
|
18
|
+
|
16
19
|
def self.logger=(logger)
|
17
20
|
@logger = logger
|
18
21
|
end
|
@@ -37,16 +40,20 @@ module RailsFailover
|
|
37
40
|
@on_failover_callback = block
|
38
41
|
end
|
39
42
|
|
40
|
-
def self.on_failover_callback
|
41
|
-
@on_failover_callback
|
43
|
+
def self.on_failover_callback!(key)
|
44
|
+
@on_failover_callback&.call(key)
|
45
|
+
rescue => e
|
46
|
+
logger.warn("RailsFailover::Redis.on_failover failed: #{e.class} #{e.message}\n#{e.backtrace.join("\n")}")
|
42
47
|
end
|
43
48
|
|
44
49
|
def self.on_fallback(&block)
|
45
50
|
@on_fallback_callback = block
|
46
51
|
end
|
47
52
|
|
48
|
-
def self.on_fallback_callback
|
49
|
-
@on_fallback_callback
|
53
|
+
def self.on_fallback_callback!(key)
|
54
|
+
@on_fallback_callback&.call(key)
|
55
|
+
rescue => e
|
56
|
+
logger.warn("RailsFailover::Redis.on_fallback failed: #{e.class} #{e.message}\n#{e.backtrace.join("\n")}")
|
50
57
|
end
|
51
58
|
|
52
59
|
# For testing
|
@@ -10,7 +10,11 @@ module RailsFailover
|
|
10
10
|
|
11
11
|
options[:driver] = Class.new(options[:driver]) do
|
12
12
|
def self.connect(options)
|
13
|
-
|
13
|
+
is_failover_replica = (options[:host] == options[:replica_host]) &&
|
14
|
+
(options[:port] == options[:replica_port])
|
15
|
+
super(options).tap do |conn|
|
16
|
+
conn.rails_failover_role = is_failover_replica ? REPLICA : PRIMARY
|
17
|
+
end
|
14
18
|
rescue ::Redis::TimeoutError,
|
15
19
|
SocketError,
|
16
20
|
Errno::EADDRNOTAVAIL,
|
@@ -22,9 +26,16 @@ module RailsFailover
|
|
22
26
|
Errno::ETIMEDOUT,
|
23
27
|
Errno::EINVAL => e
|
24
28
|
|
25
|
-
Handler.instance.verify_primary(options)
|
29
|
+
Handler.instance.verify_primary(options) if !is_failover_replica
|
26
30
|
raise e
|
27
31
|
end
|
32
|
+
|
33
|
+
attr_accessor :rails_failover_role
|
34
|
+
|
35
|
+
def shutdown_socket
|
36
|
+
@sock&.shutdown
|
37
|
+
rescue Errno::ENOTCONN
|
38
|
+
end
|
28
39
|
end
|
29
40
|
|
30
41
|
options[:original_driver] = orignal_driver
|
@@ -54,8 +65,8 @@ module RailsFailover
|
|
54
65
|
|
55
66
|
def replica_options(options)
|
56
67
|
opts = options.dup
|
57
|
-
opts[:host] = opts
|
58
|
-
opts[:port] = opts
|
68
|
+
opts[:host] = opts[:replica_host]
|
69
|
+
opts[:port] = opts[:replica_port]
|
59
70
|
opts
|
60
71
|
end
|
61
72
|
end
|
@@ -2,6 +2,7 @@
|
|
2
2
|
|
3
3
|
require 'monitor'
|
4
4
|
require 'singleton'
|
5
|
+
require 'concurrent'
|
5
6
|
|
6
7
|
module RailsFailover
|
7
8
|
class Redis
|
@@ -12,58 +13,65 @@ module RailsFailover
|
|
12
13
|
PRIMARY_ROLE_STATUS = "role:master"
|
13
14
|
PRIMARY_LOADED_STATUS = "loading:0"
|
14
15
|
VERIFY_FREQUENCY_BUFFER_PRECENT = 20
|
16
|
+
SOFT_DISCONNECT_TIMEOUT_SECONDS = 1
|
17
|
+
SOFT_DISCONNECT_POLL_SECONDS = 0.05
|
15
18
|
|
16
19
|
def initialize
|
17
|
-
@primaries_down =
|
18
|
-
@clients =
|
19
|
-
@ancestor_pid = Process.pid
|
20
|
+
@primaries_down = Concurrent::Map.new
|
21
|
+
@clients = Concurrent::Map.new
|
20
22
|
|
21
23
|
super() # Monitor#initialize
|
22
24
|
end
|
23
25
|
|
24
26
|
def verify_primary(options)
|
25
|
-
|
26
|
-
primary_down(options)
|
27
|
-
disconnect_clients(options)
|
27
|
+
primary_down(options)
|
28
28
|
|
29
|
+
mon_synchronize do
|
29
30
|
return if @thread&.alive?
|
30
|
-
|
31
31
|
logger&.warn "Failover for Redis has been initiated"
|
32
|
+
@thread = Thread.new { loop_until_all_up }
|
33
|
+
end
|
34
|
+
end
|
32
35
|
|
33
|
-
|
34
|
-
|
35
|
-
|
36
|
-
|
37
|
-
|
36
|
+
def register_client(client)
|
37
|
+
id = client.options[:id]
|
38
|
+
clients_for_id(id).put_if_absent(client, true)
|
39
|
+
end
|
40
|
+
|
41
|
+
def deregister_client(client)
|
42
|
+
id = client.options[:id]
|
43
|
+
clients_for_id(id).delete(client)
|
44
|
+
end
|
45
|
+
|
46
|
+
def primary_down?(options)
|
47
|
+
primaries_down[options[:id]]
|
48
|
+
end
|
38
49
|
|
39
|
-
|
40
|
-
|
41
|
-
|
42
|
-
thread.join
|
50
|
+
def primaries_down_count
|
51
|
+
primaries_down.size
|
52
|
+
end
|
43
53
|
|
44
|
-
|
45
|
-
logger&.warn "Fallback to primary for Redis has been completed."
|
54
|
+
private
|
46
55
|
|
47
|
-
|
48
|
-
|
49
|
-
|
50
|
-
|
51
|
-
end
|
56
|
+
def loop_until_all_up
|
57
|
+
loop do
|
58
|
+
ensure_primary_clients_disconnected
|
59
|
+
try_fallback_to_primary
|
52
60
|
|
53
|
-
|
54
|
-
|
55
|
-
|
61
|
+
if all_primaries_up
|
62
|
+
logger&.warn "Fallback to primary for Redis has been completed."
|
63
|
+
break
|
56
64
|
end
|
57
65
|
end
|
58
66
|
end
|
59
67
|
|
60
|
-
def
|
68
|
+
def try_fallback_to_primary
|
61
69
|
frequency = RailsFailover::Redis.verify_primary_frequency_seconds
|
62
70
|
sleep(frequency * ((rand(VERIFY_FREQUENCY_BUFFER_PRECENT) + 100) / 100.0))
|
63
71
|
|
64
72
|
active_primaries_keys = {}
|
65
73
|
|
66
|
-
|
74
|
+
primaries_down.each do |key, options|
|
67
75
|
info = nil
|
68
76
|
options = options.dup
|
69
77
|
|
@@ -86,110 +94,104 @@ module RailsFailover
|
|
86
94
|
|
87
95
|
active_primaries_keys.each do |key, options|
|
88
96
|
primary_up(options)
|
89
|
-
disconnect_clients(options)
|
97
|
+
disconnect_clients(options, RailsFailover::Redis::REPLICA)
|
90
98
|
end
|
91
99
|
end
|
92
100
|
|
93
|
-
def
|
94
|
-
|
95
|
-
|
96
|
-
mon_synchronize do
|
97
|
-
clients[key] ||= []
|
98
|
-
clients[key] << client
|
99
|
-
end
|
101
|
+
def all_primaries_up
|
102
|
+
primaries_down.empty?
|
100
103
|
end
|
101
104
|
|
102
|
-
def
|
103
|
-
|
105
|
+
def primary_up(options)
|
106
|
+
already_up = !primaries_down.delete(options[:id])
|
107
|
+
RailsFailover::Redis.on_fallback_callback!(options[:id]) if !already_up
|
108
|
+
end
|
104
109
|
|
105
|
-
|
106
|
-
|
107
|
-
|
110
|
+
def primary_down(options)
|
111
|
+
already_down = primaries_down.put_if_absent(options[:id], options.dup)
|
112
|
+
RailsFailover::Redis.on_failover_callback!(options[:id]) if !already_down
|
113
|
+
end
|
108
114
|
|
109
|
-
|
110
|
-
|
111
|
-
|
112
|
-
|
115
|
+
def primaries_down
|
116
|
+
ancestor_pids = nil
|
117
|
+
value = @primaries_down.compute_if_absent(Process.pid) do
|
118
|
+
ancestor_pids = @primaries_down.keys
|
119
|
+
@primaries_down.values.first || Concurrent::Map.new
|
113
120
|
end
|
114
|
-
end
|
115
121
|
|
116
|
-
|
117
|
-
|
118
|
-
primaries_down[options[:id]]
|
122
|
+
ancestor_pids&.each do |pid|
|
123
|
+
@primaries_down.delete(pid)&.each { |id, options| verify_primary(options) }
|
119
124
|
end
|
120
|
-
end
|
121
125
|
|
122
|
-
|
126
|
+
value
|
127
|
+
end
|
123
128
|
|
124
|
-
def
|
125
|
-
|
129
|
+
def clients_for_id(id)
|
130
|
+
clients.compute_if_absent(id) { Concurrent::Map.new }
|
126
131
|
end
|
127
132
|
|
128
|
-
def
|
129
|
-
|
130
|
-
|
133
|
+
def clients
|
134
|
+
ancestor_pids = nil
|
135
|
+
clients_for_pid = @clients.compute_if_absent(Process.pid) do
|
136
|
+
ancestor_pids = @clients.keys
|
137
|
+
Concurrent::Map.new
|
131
138
|
end
|
139
|
+
ancestor_pids&.each { |k| @clients.delete(k) }
|
140
|
+
clients_for_pid
|
132
141
|
end
|
133
142
|
|
134
|
-
def
|
135
|
-
|
136
|
-
|
143
|
+
def ensure_primary_clients_disconnected
|
144
|
+
primaries_down.each do |key, options|
|
145
|
+
disconnect_clients(options, RailsFailover::Redis::PRIMARY)
|
137
146
|
end
|
138
147
|
end
|
139
148
|
|
140
|
-
def
|
141
|
-
|
142
|
-
return @clients[process_pid] if @clients[process_pid]
|
149
|
+
def disconnect_clients(options, role)
|
150
|
+
id = options[:id]
|
143
151
|
|
144
|
-
|
145
|
-
|
146
|
-
|
152
|
+
matched_clients = clients_for_id(id)&.keys
|
153
|
+
&.filter { |c| c.connection.rails_failover_role == role }
|
154
|
+
&.to_set
|
147
155
|
|
148
|
-
|
149
|
-
@clients.delete(@ancestor_pid)
|
150
|
-
end
|
151
|
-
end
|
156
|
+
return if matched_clients.nil? || matched_clients.empty?
|
152
157
|
|
153
|
-
|
158
|
+
# This is not ideal, but the mutex we need is contained
|
159
|
+
# in the ::Redis instance, not the Redis::Client
|
160
|
+
ObjectSpace.each_object(::Redis) do |redis|
|
161
|
+
# When subscribed, Redis#_client is not a Redis::Client
|
162
|
+
# Instance variable is the only reliable way
|
163
|
+
client = redis.instance_variable_get(:@original_client)
|
164
|
+
next if !matched_clients.include?(client)
|
165
|
+
soft_disconnect(redis, client, role)
|
154
166
|
end
|
155
167
|
end
|
156
168
|
|
157
|
-
|
158
|
-
|
159
|
-
|
169
|
+
# Calling .disconnect can cause a running subscribe() to block forever
|
170
|
+
# Therefore try to acquire the lock
|
171
|
+
def soft_disconnect(redis, client, role)
|
172
|
+
has_lock = redis.mon_try_enter
|
160
173
|
|
161
|
-
|
162
|
-
|
163
|
-
|
164
|
-
|
165
|
-
|
166
|
-
@primaries_down.delete(@ancestor_pid)&.each do |id, options|
|
167
|
-
verify_primary(options)
|
168
|
-
end
|
169
|
-
end
|
174
|
+
if !has_lock
|
175
|
+
begin
|
176
|
+
client.connection.shutdown_socket
|
177
|
+
rescue => e
|
178
|
+
logger&.warn "Redis shutdown_socket for (#{role}) failed with #{e.class} '#{e.message}'"
|
170
179
|
end
|
171
180
|
|
172
|
-
|
173
|
-
|
174
|
-
|
175
|
-
|
176
|
-
|
177
|
-
|
178
|
-
|
179
|
-
|
180
|
-
if to_disconnect = clients[key].dup
|
181
|
-
# Don't disconnect connections abruptly since it may lead to unexepcted
|
182
|
-
# errors. Is there a better way to do this without having to monkey patch
|
183
|
-
# the redis-rb gem heavily?
|
184
|
-
ObjectSpace.each_object(::Redis).each do |redis|
|
185
|
-
to_disconnect.each do |c|
|
186
|
-
if redis._client == c
|
187
|
-
redis.synchronize { |_client| _client.disconnect }
|
188
|
-
end
|
189
|
-
end
|
190
|
-
end
|
181
|
+
waiting_since = Process.clock_gettime(Process::CLOCK_MONOTONIC)
|
182
|
+
loop do # Keep trying
|
183
|
+
break if has_lock = redis.mon_try_enter
|
184
|
+
break if !client.connection.connected? # Disconnected by other thread
|
185
|
+
break if client.connection.rails_failover_role != role # Reconnected by other thread
|
186
|
+
time_now = Process.clock_gettime(Process::CLOCK_MONOTONIC)
|
187
|
+
break if time_now > waiting_since + SOFT_DISCONNECT_TIMEOUT_SECONDS
|
188
|
+
sleep SOFT_DISCONNECT_POLL_SECONDS
|
191
189
|
end
|
192
190
|
end
|
191
|
+
|
192
|
+
client.disconnect if client.connection&.rails_failover_role == role
|
193
|
+
ensure
|
194
|
+
redis.mon_exit if has_lock
|
193
195
|
end
|
194
196
|
|
195
197
|
def logger
|
data/makefile
CHANGED
@@ -18,4 +18,4 @@ stop_dummy_rails_server:
|
|
18
18
|
@kill -TERM $(shell cat spec/support/dummy_app/tmp/pids/unicorn.pid)
|
19
19
|
|
20
20
|
teardown_dummy_rails_server:
|
21
|
-
@cd spec/support/dummy_app && DISABLE_DATABASE_ENVIRONMENT_CHECK=1 RAILS_ENV=production $(BUNDLER_BIN) exec rails db:drop
|
21
|
+
@cd spec/support/dummy_app && (! (bundle check > /dev/null 2>&1) || DISABLE_DATABASE_ENVIRONMENT_CHECK=1 RAILS_ENV=production $(BUNDLER_BIN) exec rails db:drop)
|
data/rails_failover.gemspec
CHANGED
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: rails_failover
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.6.2
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Alan Tan
|
8
8
|
autorequire:
|
9
9
|
bindir: exe
|
10
10
|
cert_chain: []
|
11
|
-
date: 2020-
|
11
|
+
date: 2020-11-25 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: activerecord
|
@@ -38,6 +38,20 @@ dependencies:
|
|
38
38
|
- - "~>"
|
39
39
|
- !ruby/object:Gem::Version
|
40
40
|
version: '6.0'
|
41
|
+
- !ruby/object:Gem::Dependency
|
42
|
+
name: concurrent-ruby
|
43
|
+
requirement: !ruby/object:Gem::Requirement
|
44
|
+
requirements:
|
45
|
+
- - ">="
|
46
|
+
- !ruby/object:Gem::Version
|
47
|
+
version: '0'
|
48
|
+
type: :runtime
|
49
|
+
prerelease: false
|
50
|
+
version_requirements: !ruby/object:Gem::Requirement
|
51
|
+
requirements:
|
52
|
+
- - ">="
|
53
|
+
- !ruby/object:Gem::Version
|
54
|
+
version: '0'
|
41
55
|
description:
|
42
56
|
email:
|
43
57
|
- tgx@discourse.org
|
@@ -45,10 +59,10 @@ executables: []
|
|
45
59
|
extensions: []
|
46
60
|
extra_rdoc_files: []
|
47
61
|
files:
|
62
|
+
- ".github/workflows/ci.yml"
|
48
63
|
- ".gitignore"
|
49
64
|
- ".rspec"
|
50
65
|
- ".rubocop.yml"
|
51
|
-
- ".travis.yml"
|
52
66
|
- CHANGELOG.md
|
53
67
|
- CODE_OF_CONDUCT.md
|
54
68
|
- Gemfile
|
@@ -92,7 +106,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
92
106
|
- !ruby/object:Gem::Version
|
93
107
|
version: '0'
|
94
108
|
requirements: []
|
95
|
-
rubygems_version: 3.
|
109
|
+
rubygems_version: 3.0.3
|
96
110
|
signing_key:
|
97
111
|
specification_version: 4
|
98
112
|
summary: Failover for ActiveRecord and Redis
|