rails_failover 0.8.1 → 1.0.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.github/workflows/ci.yml +55 -30
- data/.rubocop.yml +1 -0
- data/.streerc +2 -0
- data/CHANGELOG.md +11 -1
- data/Gemfile +0 -9
- data/README.md +10 -10
- data/lib/rails_failover/active_record/handler.rb +11 -12
- data/lib/rails_failover/active_record/middleware.rb +32 -27
- data/lib/rails_failover/active_record/railtie.rb +11 -8
- data/lib/rails_failover/active_record.rb +27 -9
- data/lib/rails_failover/redis/connector.rb +8 -14
- data/lib/rails_failover/redis/handler.rb +17 -16
- data/lib/rails_failover/redis.rb +9 -5
- data/lib/rails_failover/version.rb +1 -1
- data/lib/rails_failover.rb +2 -1
- data/lib/redis/patches/client.rb +1 -1
- data/makefile +3 -3
- data/rails_failover.gemspec +25 -15
- metadata +130 -3
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 51e57db306d29440ba74abe00ee001dc4be39121c552f3f7797cb7d0c5bfbba0
|
4
|
+
data.tar.gz: 10dc1a892042c6432ccd196905d744d09235abe3887cc039c46a75a1eafebcb6
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 8043b0080a388a26d90c06c0887455ff1e1630892beafea4171243e9aa17552b83db7d2523e95da6787c1178f883574e32b43dcc8f290e1aa1dbb0ffef153271
|
7
|
+
data.tar.gz: 2b604f3a8c3c4a5e2d3e33098ead78a0c426c3be6e00f9560c2753e277369f46f25523dcc20e59be664af7c59d3a82f8979f3e05bb8bb034be06f33393341835
|
data/.github/workflows/ci.yml
CHANGED
@@ -4,72 +4,97 @@ on:
|
|
4
4
|
pull_request:
|
5
5
|
push:
|
6
6
|
branches:
|
7
|
-
- master
|
8
7
|
- main
|
9
8
|
|
10
9
|
jobs:
|
11
|
-
|
10
|
+
lint:
|
12
11
|
runs-on: ubuntu-latest
|
13
12
|
|
14
|
-
|
15
|
-
|
13
|
+
steps:
|
14
|
+
- uses: actions/checkout@v3
|
15
|
+
|
16
|
+
- name: Setup ruby
|
17
|
+
uses: ruby/setup-ruby@v1
|
18
|
+
with:
|
19
|
+
ruby-version: '3.2'
|
20
|
+
bundler-cache: true
|
21
|
+
|
22
|
+
- name: Rubocop
|
23
|
+
run: bundle exec rubocop
|
24
|
+
|
25
|
+
- name: syntax_tree
|
26
|
+
if: ${{ !cancelled() }}
|
27
|
+
run: |
|
28
|
+
set -E
|
29
|
+
bundle exec stree check Gemfile rails_failover.gemspec $(git ls-files '*.rb')
|
30
|
+
|
31
|
+
redis:
|
32
|
+
name: 'Redis (Ruby ${{ matrix.ruby }})'
|
33
|
+
runs-on: ubuntu-latest
|
16
34
|
|
17
35
|
strategy:
|
18
36
|
fail-fast: false
|
19
37
|
matrix:
|
20
|
-
ruby: [
|
21
|
-
build_types: ["REDIS", "ACTIVERECORD"]
|
22
|
-
include:
|
23
|
-
- ruby: "3.1"
|
24
|
-
build_types: "LINT"
|
38
|
+
ruby: ['2.7', '3.0', '3.1', '3.2']
|
25
39
|
|
26
40
|
steps:
|
27
|
-
- uses: actions/checkout@
|
41
|
+
- uses: actions/checkout@v3
|
28
42
|
|
29
43
|
- name: Setup ruby
|
30
44
|
uses: ruby/setup-ruby@v1
|
31
45
|
with:
|
32
46
|
ruby-version: ${{ matrix.ruby }}
|
33
|
-
|
34
|
-
- name: Setup bundler
|
35
|
-
run: gem install bundler
|
36
|
-
|
37
|
-
- name: Setup gems
|
38
|
-
run: bundle install
|
39
|
-
|
40
|
-
- name: Rubocop
|
41
|
-
run: bundle exec rubocop
|
42
|
-
if: env.BUILD_TYPE == 'LINT'
|
47
|
+
bundler-cache: true
|
43
48
|
|
44
49
|
- name: Setup redis
|
45
50
|
run: sudo apt-get install redis-server
|
46
|
-
if: env.BUILD_TYPE == 'REDIS'
|
47
51
|
|
48
52
|
- name: Redis specs
|
49
53
|
run: bin/rspec redis
|
50
|
-
if: env.BUILD_TYPE == 'REDIS'
|
51
54
|
|
52
|
-
|
53
|
-
|
54
|
-
|
55
|
+
active_record:
|
56
|
+
runs-on: ubuntu-latest
|
57
|
+
name: 'ActiveRecord ~>${{ matrix.rails }} (Ruby ${{ matrix.ruby }})'
|
58
|
+
|
59
|
+
strategy:
|
60
|
+
fail-fast: false
|
61
|
+
matrix:
|
62
|
+
ruby: ['3.2', '3.1', '3.0', '2.7']
|
63
|
+
rails: ['7.0.0']
|
64
|
+
include:
|
65
|
+
- ruby: '3.2'
|
66
|
+
rails: '6.1.0'
|
67
|
+
- ruby: '3.2'
|
68
|
+
rails: '6.0.0'
|
69
|
+
|
70
|
+
steps:
|
71
|
+
- uses: actions/checkout@v3
|
72
|
+
|
73
|
+
- name: Setup ruby
|
74
|
+
uses: ruby/setup-ruby@v1
|
75
|
+
with:
|
76
|
+
ruby-version: ${{ matrix.ruby }}
|
77
|
+
|
78
|
+
- name: Setup gems
|
79
|
+
run: bundle install
|
55
80
|
|
56
81
|
- name: Setup postgres
|
57
82
|
run: |
|
58
83
|
make setup_pg
|
59
84
|
make start_pg
|
60
|
-
if: env.BUILD_TYPE == 'ACTIVERECORD'
|
61
85
|
|
62
86
|
- name: ActiveRecord specs
|
87
|
+
env:
|
88
|
+
RAILS_VERSION: ${{ matrix.rails }}
|
63
89
|
run: bin/rspec active_record
|
64
|
-
if: env.BUILD_TYPE == 'ACTIVERECORD'
|
65
90
|
|
66
91
|
publish:
|
67
|
-
if: github.event_name == 'push' &&
|
68
|
-
needs:
|
92
|
+
if: github.event_name == 'push' && github.ref == 'refs/heads/main'
|
93
|
+
needs: [lint, redis, active_record]
|
69
94
|
runs-on: ubuntu-latest
|
70
95
|
|
71
96
|
steps:
|
72
|
-
- uses: actions/checkout@
|
97
|
+
- uses: actions/checkout@v3
|
73
98
|
|
74
99
|
- name: Release Gem
|
75
100
|
uses: discourse/publish-rubygems-action@v2
|
data/.rubocop.yml
CHANGED
data/.streerc
ADDED
data/CHANGELOG.md
CHANGED
@@ -1,4 +1,5 @@
|
|
1
1
|
# Changelog
|
2
|
+
|
2
3
|
All notable changes to this project will be documented in this file.
|
3
4
|
|
4
5
|
The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
|
@@ -6,6 +7,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
|
|
6
7
|
|
7
8
|
## [Unreleased]
|
8
9
|
|
10
|
+
## [1.0.0] - 2023-04-07
|
11
|
+
|
12
|
+
- DEV: Remove the support for Ruby < 2.7
|
13
|
+
- DEV: Compatibility with Rails 7.1+
|
14
|
+
|
9
15
|
## [0.8.0] - 2022-01-17
|
10
16
|
|
11
17
|
- FEATURE: Compatibility with Rails 7.0+
|
@@ -20,7 +26,7 @@ No changes.
|
|
20
26
|
|
21
27
|
## [0.7.1] - 2021-04-14
|
22
28
|
|
23
|
-
- FIX: Backward
|
29
|
+
- FIX: Backward compatibility with Rails 6.0
|
24
30
|
|
25
31
|
## [0.7.0] - 2021-04-14
|
26
32
|
|
@@ -49,6 +55,7 @@ No changes.
|
|
49
55
|
Previously, a replica failing would cause it to be added to the 'primaries_down' list. The fallback handler would then continuously try and fallback the replica to itself, looping forever, and meaning that fallback to primary would never happen.
|
50
56
|
|
51
57
|
## [0.6.0] - 2020-11-09
|
58
|
+
|
52
59
|
- FEATURE: Run failover/fallback callbacks once for each backend
|
53
60
|
|
54
61
|
Previously the failover callback would only fire when the first backend failed, and the fallback callback would only fire when the last backend recovered. Now both failover and fallback callbacks will be triggered for each backend. The key for each backend is also passed to the callbacks for consumption by consuming applications.
|
@@ -58,6 +65,7 @@ No changes.
|
|
58
65
|
This is intended for consumption by monitoring systems (e.g. the Discourse prometheus exporter)
|
59
66
|
|
60
67
|
## [0.5.9] - 2020-11-06
|
68
|
+
|
61
69
|
- FIX: Ignore errors from the redis socket shutdown call
|
62
70
|
|
63
71
|
This can fail with various i/o errors, but in all cases we want the thread to continue closing the connection with the error, and all the other connections.
|
@@ -67,6 +75,7 @@ No changes.
|
|
67
75
|
- FIX: Handle concurrency issues during redis disconnection (#10)
|
68
76
|
|
69
77
|
This handles concurrency issues which can happen during redis failover/fallback:
|
78
|
+
|
70
79
|
- Previously, 'subscribed' redis clients were skipped during the disconnect process. This is resolved by directly accessing the original_client from the ::Redis instance
|
71
80
|
- Trying to acquire the mutex on a subscribed redis client is impossible, so the close operation would never complete. Now we send the shutdown() signal to the thread, then allow up to 1 second for the mutex to be released before we close the socket
|
72
81
|
- Failover is almost always triggered inside a redis client mutex. Failover then has its own mutex, within which we attempted to acquire mutexes for all redis clients. This logic causes a deadlock when multiple clients failover simultaneously. Now, all disconnection is performed by the Redis::Handler failover thread, outside of any other mutexes. To make this safe, the primary/replica state is stored in the connection driver, and disconnect_clients is updated to specifically target primary/replica connections.
|
@@ -107,4 +116,5 @@ No changes.
|
|
107
116
|
## [0.5.2] - 2020-06-23
|
108
117
|
|
109
118
|
### Changed
|
119
|
+
|
110
120
|
- FIX: Only rescue from connection errors.
|
data/Gemfile
CHANGED
@@ -3,12 +3,3 @@ source "https://rubygems.org"
|
|
3
3
|
|
4
4
|
# Specify your gem's dependencies in rails_failover.gemspec
|
5
5
|
gemspec
|
6
|
-
|
7
|
-
gem "rake", "~> 12.0"
|
8
|
-
gem "rspec", "~> 3.0"
|
9
|
-
gem 'rubocop-discourse'
|
10
|
-
gem 'byebug'
|
11
|
-
gem 'redis', '~> 4.1'
|
12
|
-
gem 'pg', '~> 1.2'
|
13
|
-
gem 'activerecord', '~> 6.0'
|
14
|
-
gem 'rack'
|
data/README.md
CHANGED
@@ -3,7 +3,7 @@
|
|
3
3
|
Automatic failover and recovery for primary/replica setup for:
|
4
4
|
|
5
5
|
1. Redis
|
6
|
-
|
6
|
+
2. ActiveRecord (PostgreSQL/MySQL Adapters)
|
7
7
|
|
8
8
|
## Installation
|
9
9
|
|
@@ -29,7 +29,7 @@ In `config/application.rb` add `require 'rails_failover/active_record'` after `r
|
|
29
29
|
|
30
30
|
In your database configuration, simply add `replica_host` and `replica_port` to your database configuration.
|
31
31
|
|
32
|
-
```
|
32
|
+
```yml
|
33
33
|
production:
|
34
34
|
host: <primary db server host>
|
35
35
|
port: <primary db server port>
|
@@ -37,11 +37,11 @@ production:
|
|
37
37
|
replica_port: <replica db server port>
|
38
38
|
```
|
39
39
|
|
40
|
-
The gem will automatically create an `ActiveRecord::ConnectionAdapters::ConnectionHandler` with the `ActiveRecord
|
40
|
+
The gem will automatically create an `ActiveRecord::ConnectionAdapters::ConnectionHandler` with the `ActiveRecord.reading_role` as the `handler_key`.
|
41
41
|
|
42
42
|
#### Failover/Fallback Hooks
|
43
43
|
|
44
|
-
```
|
44
|
+
```ruby
|
45
45
|
RailsFailover::ActiveRecord.on_failover do
|
46
46
|
# Enable readonly mode
|
47
47
|
end
|
@@ -55,7 +55,7 @@ end
|
|
55
55
|
|
56
56
|
Note: This API is unstable and is likely to change when Rails 6.1 is released with sharding support.
|
57
57
|
|
58
|
-
```
|
58
|
+
```yml
|
59
59
|
# config/database.yml
|
60
60
|
|
61
61
|
production:
|
@@ -79,13 +79,13 @@ connects_to database: { writing: :primary, second_database_writing: :second_data
|
|
79
79
|
|
80
80
|
Add `require 'rails_failover/redis'` before creating a `Redis` instance.
|
81
81
|
|
82
|
-
```
|
83
|
-
Redis.new(host: "127.0.0.1", port: 6379, replica_host: "127.0.0.1", replica_port: 6380, connector: RailsFailover::Redis::Connector)
|
82
|
+
```ruby
|
83
|
+
Redis.new(host: "127.0.0.1", port: 6379, replica_host: "127.0.0.1", replica_port: 6380, connector: RailsFailover::Redis::Connector)
|
84
84
|
```
|
85
85
|
|
86
86
|
Callbacks can be registered when the primary connection is down and when it is up.
|
87
87
|
|
88
|
-
```
|
88
|
+
```ruby
|
89
89
|
RailsFailover::Redis.on_failover_callback do
|
90
90
|
# Switch site to read-only mode
|
91
91
|
end
|
@@ -108,8 +108,8 @@ To install this gem onto your local machine, run `bundle exec rake install`. To
|
|
108
108
|
The ActiveRecord failover tests are run against a dummy Rails server. Run the following commands to run the test:
|
109
109
|
|
110
110
|
1. `make setup_pg`
|
111
|
-
|
112
|
-
|
111
|
+
2. `make start_pg`
|
112
|
+
3. `bin/rspec active_record`. You may also run the tests with more unicorn workers by adding the `UNICORN_WORKERS` env variable.
|
113
113
|
|
114
114
|
#### Redis
|
115
115
|
|
@@ -1,7 +1,7 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
|
-
require
|
3
|
-
require
|
4
|
-
require
|
2
|
+
require "singleton"
|
3
|
+
require "monitor"
|
4
|
+
require "concurrent"
|
5
5
|
|
6
6
|
module RailsFailover
|
7
7
|
module ActiveRecord
|
@@ -9,7 +9,7 @@ module RailsFailover
|
|
9
9
|
include Singleton
|
10
10
|
include MonitorMixin
|
11
11
|
|
12
|
-
|
12
|
+
VERIFY_FREQUENCY_BUFFER_PERCENT = 20
|
13
13
|
|
14
14
|
def initialize
|
15
15
|
@primaries_down = Concurrent::Map.new
|
@@ -50,7 +50,7 @@ module RailsFailover
|
|
50
50
|
|
51
51
|
def initiate_fallback_to_primary
|
52
52
|
frequency = RailsFailover::ActiveRecord.verify_primary_frequency_seconds
|
53
|
-
sleep(frequency * ((rand(
|
53
|
+
sleep(frequency * ((rand(VERIFY_FREQUENCY_BUFFER_PERCENT) + 100) / 100.0))
|
54
54
|
|
55
55
|
active_handler_keys = []
|
56
56
|
|
@@ -83,9 +83,7 @@ module RailsFailover
|
|
83
83
|
end
|
84
84
|
end
|
85
85
|
|
86
|
-
active_handler_keys.each
|
87
|
-
primary_up(handler_key)
|
88
|
-
end
|
86
|
+
active_handler_keys.each { |handler_key| primary_up(handler_key) }
|
89
87
|
end
|
90
88
|
|
91
89
|
def all_primaries_up
|
@@ -108,10 +106,11 @@ module RailsFailover
|
|
108
106
|
|
109
107
|
def primaries_down
|
110
108
|
ancestor_pids = nil
|
111
|
-
value =
|
112
|
-
|
113
|
-
|
114
|
-
|
109
|
+
value =
|
110
|
+
@primaries_down.compute_if_absent(Process.pid) do
|
111
|
+
ancestor_pids = @primaries_down.keys
|
112
|
+
@primaries_down.values.first || Concurrent::Map.new
|
113
|
+
end
|
115
114
|
|
116
115
|
ancestor_pids&.each do |pid|
|
117
116
|
@primaries_down.delete(pid)&.each_key { |key| verify_primary(key) }
|
@@ -4,20 +4,18 @@ module RailsFailover
|
|
4
4
|
module ActiveRecord
|
5
5
|
class Interceptor
|
6
6
|
def self.adapter_errors
|
7
|
-
@adapter_errors ||=
|
8
|
-
|
9
|
-
|
10
|
-
|
11
|
-
|
7
|
+
@adapter_errors ||=
|
8
|
+
begin
|
9
|
+
if defined?(::PG)
|
10
|
+
[::PG::UnableToSend, ::PG::ConnectionBad]
|
11
|
+
elsif defined?(::Mysql2)
|
12
|
+
[::Mysql2::Error::ConnectionError]
|
13
|
+
end
|
12
14
|
end
|
13
|
-
end
|
14
15
|
end
|
15
16
|
|
16
17
|
def self.handle(request, exception)
|
17
|
-
verify_primary(
|
18
|
-
exception,
|
19
|
-
request.env[Middleware::WRITING_ROLE_HEADER]
|
20
|
-
)
|
18
|
+
verify_primary(exception, request.env[Middleware::WRITING_ROLE_HEADER])
|
21
19
|
end
|
22
20
|
|
23
21
|
def self.verify_primary(exception, writing_role)
|
@@ -29,11 +27,7 @@ module RailsFailover
|
|
29
27
|
end
|
30
28
|
|
31
29
|
def self.resolve_cause(exception)
|
32
|
-
|
33
|
-
resolve_cause(exception.cause)
|
34
|
-
else
|
35
|
-
exception
|
36
|
-
end
|
30
|
+
exception.cause ? resolve_cause(exception.cause) : exception
|
37
31
|
end
|
38
32
|
end
|
39
33
|
|
@@ -50,14 +44,19 @@ module RailsFailover
|
|
50
44
|
end
|
51
45
|
|
52
46
|
def call(env)
|
53
|
-
current_role = ::ActiveRecord::Base.current_role || ::ActiveRecord
|
54
|
-
is_writing_role = current_role.to_s.end_with?(::ActiveRecord
|
47
|
+
current_role = ::ActiveRecord::Base.current_role || RailsFailover::ActiveRecord.writing_role
|
48
|
+
is_writing_role = current_role.to_s.end_with?(RailsFailover::ActiveRecord.writing_role.to_s)
|
55
49
|
writing_role = resolve_writing_role(current_role, is_writing_role)
|
56
50
|
|
57
51
|
role =
|
58
|
-
if primary_down =
|
52
|
+
if primary_down =
|
53
|
+
self.class.force_reading_role_callback&.call(env) ||
|
54
|
+
Handler.instance.primary_down?(writing_role)
|
59
55
|
reading_role = resolve_reading_role(current_role, is_writing_role)
|
60
|
-
ensure_reading_connection_established!(
|
56
|
+
ensure_reading_connection_established!(
|
57
|
+
writing_role: writing_role,
|
58
|
+
reading_role: reading_role,
|
59
|
+
)
|
61
60
|
reading_role
|
62
61
|
else
|
63
62
|
writing_role
|
@@ -96,19 +95,25 @@ module RailsFailover
|
|
96
95
|
if is_writing_role
|
97
96
|
current_role
|
98
97
|
else
|
99
|
-
current_role
|
100
|
-
|
101
|
-
|
102
|
-
|
98
|
+
current_role
|
99
|
+
.to_s
|
100
|
+
.sub(
|
101
|
+
/#{RailsFailover::ActiveRecord.reading_role}$/,
|
102
|
+
RailsFailover::ActiveRecord.writing_role.to_s,
|
103
|
+
)
|
104
|
+
.to_sym
|
103
105
|
end
|
104
106
|
end
|
105
107
|
|
106
108
|
def resolve_reading_role(current_role, is_writing_role)
|
107
109
|
if is_writing_role
|
108
|
-
current_role
|
109
|
-
|
110
|
-
|
111
|
-
|
110
|
+
current_role
|
111
|
+
.to_s
|
112
|
+
.sub(
|
113
|
+
/#{RailsFailover::ActiveRecord.writing_role}$/,
|
114
|
+
RailsFailover::ActiveRecord.reading_role.to_s,
|
115
|
+
)
|
116
|
+
.to_sym
|
112
117
|
else
|
113
118
|
current_role
|
114
119
|
end
|
@@ -4,7 +4,6 @@ module RailsFailover
|
|
4
4
|
module ActiveRecord
|
5
5
|
class Railtie < ::Rails::Railtie
|
6
6
|
initializer "rails_failover.init", after: "active_record.initialize_database" do |app|
|
7
|
-
|
8
7
|
# AR 6.0 / 6.1 compat
|
9
8
|
config =
|
10
9
|
if ::ActiveRecord::Base.respond_to? :connection_db_config
|
@@ -23,18 +22,21 @@ module RailsFailover
|
|
23
22
|
|
24
23
|
# We are doing this manually for now since we're awaiting Rails 6.1 to be released which will
|
25
24
|
# have more stable ActiveRecord APIs for handling multiple databases with different roles.
|
26
|
-
::ActiveRecord::Base.connection_handlers[
|
27
|
-
::ActiveRecord
|
25
|
+
::ActiveRecord::Base.connection_handlers[
|
26
|
+
RailsFailover::ActiveRecord.reading_role
|
27
|
+
] = ::ActiveRecord::ConnectionAdapters::ConnectionHandler.new
|
28
28
|
|
29
|
-
::ActiveRecord::Base.connection_handlers[::ActiveRecord
|
29
|
+
::ActiveRecord::Base.connection_handlers[RailsFailover::ActiveRecord.writing_role]
|
30
|
+
.connection_pools
|
31
|
+
.each do |connection_pool|
|
30
32
|
if connection_pool.respond_to?(:db_config)
|
31
33
|
config = connection_pool.db_config.configuration_hash
|
32
34
|
else
|
33
35
|
config = connection_pool.spec.config
|
34
36
|
end
|
35
37
|
RailsFailover::ActiveRecord.establish_reading_connection(
|
36
|
-
::ActiveRecord::Base.connection_handlers[::ActiveRecord
|
37
|
-
config
|
38
|
+
::ActiveRecord::Base.connection_handlers[RailsFailover::ActiveRecord.reading_role],
|
39
|
+
config,
|
38
40
|
)
|
39
41
|
end
|
40
42
|
|
@@ -43,8 +45,9 @@ module RailsFailover
|
|
43
45
|
rescue ::ActiveRecord::NoDatabaseError
|
44
46
|
# Do nothing since database hasn't been created
|
45
47
|
rescue ::PG::Error, ::ActiveRecord::ConnectionNotEstablished
|
46
|
-
Handler.instance.verify_primary(::ActiveRecord
|
47
|
-
::ActiveRecord::Base.connection_handler =
|
48
|
+
Handler.instance.verify_primary(RailsFailover::ActiveRecord.writing_role)
|
49
|
+
::ActiveRecord::Base.connection_handler =
|
50
|
+
::ActiveRecord::Base.lookup_connection_handler(:reading)
|
48
51
|
end
|
49
52
|
end
|
50
53
|
end
|
@@ -1,13 +1,20 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
|
-
require
|
3
|
+
require "active_record"
|
4
4
|
|
5
|
-
if defined?(::Rails)
|
6
|
-
|
7
|
-
|
5
|
+
require_relative "active_record/railtie" if defined?(::Rails)
|
6
|
+
|
7
|
+
require_relative "active_record/middleware"
|
8
|
+
require_relative "active_record/handler"
|
8
9
|
|
9
|
-
|
10
|
-
|
10
|
+
AR =
|
11
|
+
(
|
12
|
+
if ::ActiveRecord.respond_to?(:reading_role)
|
13
|
+
::ActiveRecord
|
14
|
+
else
|
15
|
+
::ActiveRecord::Base
|
16
|
+
end
|
17
|
+
)
|
11
18
|
|
12
19
|
module RailsFailover
|
13
20
|
module ActiveRecord
|
@@ -28,7 +35,6 @@ module RailsFailover
|
|
28
35
|
end
|
29
36
|
|
30
37
|
def self.establish_reading_connection(handler, config)
|
31
|
-
|
32
38
|
if config[:replica_host] && config[:replica_port]
|
33
39
|
replica_config = config.dup
|
34
40
|
replica_config[:host] = replica_config.delete(:replica_host)
|
@@ -49,7 +55,9 @@ module RailsFailover
|
|
49
55
|
def self.on_failover_callback!(key)
|
50
56
|
@on_failover_callback&.call(key)
|
51
57
|
rescue => e
|
52
|
-
logger.warn(
|
58
|
+
logger.warn(
|
59
|
+
"RailsFailover::ActiveRecord.on_failover failed: #{e.class} #{e.message}\n#{e.backtrace.join("\n")}",
|
60
|
+
)
|
53
61
|
end
|
54
62
|
|
55
63
|
def self.on_fallback(&block)
|
@@ -59,7 +67,17 @@ module RailsFailover
|
|
59
67
|
def self.on_fallback_callback!(key)
|
60
68
|
@on_fallback_callback&.call(key)
|
61
69
|
rescue => e
|
62
|
-
logger.warn(
|
70
|
+
logger.warn(
|
71
|
+
"RailsFailover::ActiveRecord.on_fallback failed: #{e.class} #{e.message}\n#{e.backtrace.join("\n")}",
|
72
|
+
)
|
73
|
+
end
|
74
|
+
|
75
|
+
def self.reading_role
|
76
|
+
AR.reading_role
|
77
|
+
end
|
78
|
+
|
79
|
+
def self.writing_role
|
80
|
+
AR.writing_role
|
63
81
|
end
|
64
82
|
end
|
65
83
|
end
|
@@ -1,22 +1,21 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
|
-
require_relative
|
3
|
+
require_relative "handler"
|
4
4
|
|
5
5
|
module RailsFailover
|
6
6
|
class Redis
|
7
7
|
class Connector < ::Redis::Client::Connector
|
8
8
|
def initialize(options)
|
9
|
-
|
9
|
+
original_driver = options[:driver]
|
10
10
|
options[:primary_host] = options[:host]
|
11
11
|
options[:primary_port] = options[:port]
|
12
12
|
|
13
13
|
options[:driver] = Class.new(options[:driver]) do
|
14
14
|
def self.connect(options)
|
15
|
-
is_primary =
|
16
|
-
|
17
|
-
|
18
|
-
|
19
|
-
end
|
15
|
+
is_primary =
|
16
|
+
(options[:host] == options[:primary_host]) &&
|
17
|
+
(options[:port] == options[:primary_port])
|
18
|
+
super(options).tap { |conn| conn.rails_failover_role = is_primary ? PRIMARY : REPLICA }
|
20
19
|
rescue ::Redis::TimeoutError,
|
21
20
|
SocketError,
|
22
21
|
Errno::EADDRNOTAVAIL,
|
@@ -27,7 +26,6 @@ module RailsFailover
|
|
27
26
|
Errno::ENOENT,
|
28
27
|
Errno::ETIMEDOUT,
|
29
28
|
Errno::EINVAL => e
|
30
|
-
|
31
29
|
Handler.instance.verify_primary(options) if is_primary
|
32
30
|
raise e
|
33
31
|
end
|
@@ -40,7 +38,7 @@ module RailsFailover
|
|
40
38
|
end
|
41
39
|
end
|
42
40
|
|
43
|
-
options[:original_driver] =
|
41
|
+
options[:original_driver] = original_driver
|
44
42
|
options.delete(:connector)
|
45
43
|
options[:id] ||= "#{options[:host]}:#{options[:port]}"
|
46
44
|
@replica_options = replica_options(options)
|
@@ -48,11 +46,7 @@ module RailsFailover
|
|
48
46
|
end
|
49
47
|
|
50
48
|
def resolve
|
51
|
-
|
52
|
-
@replica_options
|
53
|
-
else
|
54
|
-
@options
|
55
|
-
end
|
49
|
+
Handler.instance.primary_down?(@options) ? @replica_options : @options
|
56
50
|
end
|
57
51
|
|
58
52
|
def check(client)
|
@@ -1,8 +1,8 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
|
-
require
|
4
|
-
require
|
5
|
-
require
|
3
|
+
require "monitor"
|
4
|
+
require "singleton"
|
5
|
+
require "concurrent"
|
6
6
|
|
7
7
|
module RailsFailover
|
8
8
|
class Redis
|
@@ -12,7 +12,7 @@ module RailsFailover
|
|
12
12
|
|
13
13
|
PRIMARY_ROLE_STATUS = "role:master"
|
14
14
|
PRIMARY_LOADED_STATUS = "loading:0"
|
15
|
-
|
15
|
+
VERIFY_FREQUENCY_BUFFER_PERCENT = 20
|
16
16
|
SOFT_DISCONNECT_TIMEOUT_SECONDS = 1
|
17
17
|
SOFT_DISCONNECT_POLL_SECONDS = 0.05
|
18
18
|
|
@@ -67,7 +67,7 @@ module RailsFailover
|
|
67
67
|
|
68
68
|
def try_fallback_to_primary
|
69
69
|
frequency = RailsFailover::Redis.verify_primary_frequency_seconds
|
70
|
-
sleep(frequency * ((rand(
|
70
|
+
sleep(frequency * ((rand(VERIFY_FREQUENCY_BUFFER_PERCENT) + 100) / 100.0))
|
71
71
|
|
72
72
|
active_primaries_keys = {}
|
73
73
|
|
@@ -114,10 +114,11 @@ module RailsFailover
|
|
114
114
|
|
115
115
|
def primaries_down
|
116
116
|
ancestor_pids = nil
|
117
|
-
value =
|
118
|
-
|
119
|
-
|
120
|
-
|
117
|
+
value =
|
118
|
+
@primaries_down.compute_if_absent(Process.pid) do
|
119
|
+
ancestor_pids = @primaries_down.keys
|
120
|
+
@primaries_down.values.first || Concurrent::Map.new
|
121
|
+
end
|
121
122
|
|
122
123
|
ancestor_pids&.each do |pid|
|
123
124
|
@primaries_down.delete(pid)&.each { |id, options| verify_primary(options) }
|
@@ -132,10 +133,11 @@ module RailsFailover
|
|
132
133
|
|
133
134
|
def clients
|
134
135
|
ancestor_pids = nil
|
135
|
-
clients_for_pid =
|
136
|
-
|
137
|
-
|
138
|
-
|
136
|
+
clients_for_pid =
|
137
|
+
@clients.compute_if_absent(Process.pid) do
|
138
|
+
ancestor_pids = @clients.keys
|
139
|
+
Concurrent::Map.new
|
140
|
+
end
|
139
141
|
ancestor_pids&.each { |k| @clients.delete(k) }
|
140
142
|
clients_for_pid
|
141
143
|
end
|
@@ -149,9 +151,8 @@ module RailsFailover
|
|
149
151
|
def disconnect_clients(options, role)
|
150
152
|
id = options[:id]
|
151
153
|
|
152
|
-
matched_clients =
|
153
|
-
&.filter { |c| c.connection.rails_failover_role == role }
|
154
|
-
&.to_set
|
154
|
+
matched_clients =
|
155
|
+
clients_for_id(id)&.keys&.filter { |c| c.connection.rails_failover_role == role }&.to_set
|
155
156
|
|
156
157
|
return if matched_clients.nil? || matched_clients.empty?
|
157
158
|
|
data/lib/rails_failover/redis.rb
CHANGED
@@ -1,15 +1,15 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
|
-
require
|
3
|
+
require "redis"
|
4
4
|
|
5
|
-
supported_version =
|
5
|
+
supported_version = "4"
|
6
6
|
|
7
7
|
if Gem::Version.new(Redis::VERSION) < Gem::Version.new(supported_version)
|
8
8
|
raise "redis #{Redis::VERSION} is not supported. Please upgrade to Redis #{supported_version}."
|
9
9
|
end
|
10
10
|
|
11
11
|
require_relative "../redis/patches/client"
|
12
|
-
require_relative
|
12
|
+
require_relative "redis/connector"
|
13
13
|
|
14
14
|
module RailsFailover
|
15
15
|
class Redis
|
@@ -43,7 +43,9 @@ module RailsFailover
|
|
43
43
|
def self.on_failover_callback!(key)
|
44
44
|
@on_failover_callback&.call(key)
|
45
45
|
rescue => e
|
46
|
-
logger.warn(
|
46
|
+
logger.warn(
|
47
|
+
"RailsFailover::Redis.on_failover failed: #{e.class} #{e.message}\n#{e.backtrace.join("\n")}",
|
48
|
+
)
|
47
49
|
end
|
48
50
|
|
49
51
|
def self.on_fallback(&block)
|
@@ -53,7 +55,9 @@ module RailsFailover
|
|
53
55
|
def self.on_fallback_callback!(key)
|
54
56
|
@on_fallback_callback&.call(key)
|
55
57
|
rescue => e
|
56
|
-
logger.warn(
|
58
|
+
logger.warn(
|
59
|
+
"RailsFailover::Redis.on_fallback failed: #{e.class} #{e.message}\n#{e.backtrace.join("\n")}",
|
60
|
+
)
|
57
61
|
end
|
58
62
|
|
59
63
|
# For testing
|
data/lib/rails_failover.rb
CHANGED
data/lib/redis/patches/client.rb
CHANGED
data/makefile
CHANGED
@@ -9,13 +9,13 @@ test_active_record:
|
|
9
9
|
@ACTIVE_RECORD=1 bundle exec rspec --tag type:active_record ${RSPEC_PATH}
|
10
10
|
|
11
11
|
setup_dummy_rails_server:
|
12
|
-
@cd spec/support/dummy_app && bundle install --quiet && yarn install && RAILS_ENV=production $(BUNDLER_BIN) exec rails db:create db:migrate db:seed
|
12
|
+
@cd spec/support/dummy_app && BUNDLE_GEMFILE=Gemfile bundle install --quiet && yarn install && BUNDLE_GEMFILE=Gemfile RAILS_ENV=production $(BUNDLER_BIN) exec rails db:create db:migrate db:seed
|
13
13
|
|
14
14
|
start_dummy_rails_server:
|
15
|
-
@cd spec/support/dummy_app && BUNDLE_GEMFILE=Gemfile
|
15
|
+
@cd spec/support/dummy_app && BUNDLE_GEMFILE=Gemfile SECRET_KEY_BASE=somekey bundle exec unicorn -c config/unicorn.conf.rb -D -E production
|
16
16
|
|
17
17
|
stop_dummy_rails_server:
|
18
18
|
@kill -TERM $(shell cat spec/support/dummy_app/tmp/pids/unicorn.pid)
|
19
19
|
|
20
20
|
teardown_dummy_rails_server:
|
21
|
-
@cd spec/support/dummy_app && (! (bundle check > /dev/null 2>&1) || DISABLE_DATABASE_ENVIRONMENT_CHECK=1 RAILS_ENV=production $(BUNDLER_BIN) exec rails db:drop)
|
21
|
+
@cd spec/support/dummy_app && (! (bundle check > /dev/null 2>&1) || BUNDLE_GEMFILE=Gemfile DISABLE_DATABASE_ENVIRONMENT_CHECK=1 RAILS_ENV=production $(BUNDLER_BIN) exec rails db:drop)
|
data/rails_failover.gemspec
CHANGED
@@ -1,29 +1,39 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
|
-
require_relative
|
3
|
+
require_relative "lib/rails_failover/version"
|
4
4
|
|
5
5
|
Gem::Specification.new do |spec|
|
6
|
-
spec.name
|
7
|
-
spec.version
|
8
|
-
spec.authors
|
9
|
-
spec.email
|
6
|
+
spec.name = "rails_failover"
|
7
|
+
spec.version = RailsFailover::VERSION
|
8
|
+
spec.authors = ["Alan Tan"]
|
9
|
+
spec.email = ["tgx@discourse.org"]
|
10
10
|
|
11
|
-
spec.summary
|
12
|
-
spec.homepage
|
13
|
-
spec.license
|
14
|
-
spec.required_ruby_version = Gem::Requirement.new(">= 2.
|
11
|
+
spec.summary = "Failover for ActiveRecord and Redis"
|
12
|
+
spec.homepage = "https://github.com/discourse/rails_failover"
|
13
|
+
spec.license = "MIT"
|
14
|
+
spec.required_ruby_version = Gem::Requirement.new(">= 2.7.0")
|
15
15
|
|
16
16
|
# Specify which files should be added to the gem when it is released.
|
17
17
|
# The `git ls-files -z` loads the files in the RubyGem that have been added into git.
|
18
|
-
spec.files =
|
19
|
-
|
20
|
-
|
21
|
-
|
22
|
-
spec.
|
18
|
+
spec.files =
|
19
|
+
Dir.chdir(File.expand_path("..", __FILE__)) do
|
20
|
+
`git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
|
21
|
+
end
|
22
|
+
spec.bindir = "exe"
|
23
|
+
spec.executables = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
|
23
24
|
spec.require_paths = ["lib"]
|
24
25
|
|
25
26
|
spec.add_dependency "activerecord", "> 6.0", "< 7.1"
|
26
27
|
spec.add_dependency "railties", "> 6.0", "< 7.1"
|
27
|
-
|
28
28
|
spec.add_dependency "concurrent-ruby"
|
29
|
+
|
30
|
+
spec.add_development_dependency "rake", "~> 12.0"
|
31
|
+
spec.add_development_dependency "redis", "~> 4.1"
|
32
|
+
spec.add_development_dependency "pg", "~> 1.2"
|
33
|
+
spec.add_development_dependency "rack"
|
34
|
+
spec.add_development_dependency "rspec", "~> 3.0"
|
35
|
+
spec.add_development_dependency "byebug"
|
36
|
+
spec.add_development_dependency "rubocop-discourse"
|
37
|
+
spec.add_development_dependency "syntax_tree"
|
38
|
+
spec.add_development_dependency "syntax_tree-disable_ternary"
|
29
39
|
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: rails_failover
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 1.0.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Alan Tan
|
8
8
|
autorequire:
|
9
9
|
bindir: exe
|
10
10
|
cert_chain: []
|
11
|
-
date:
|
11
|
+
date: 2023-04-08 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: activerecord
|
@@ -64,6 +64,132 @@ dependencies:
|
|
64
64
|
- - ">="
|
65
65
|
- !ruby/object:Gem::Version
|
66
66
|
version: '0'
|
67
|
+
- !ruby/object:Gem::Dependency
|
68
|
+
name: rake
|
69
|
+
requirement: !ruby/object:Gem::Requirement
|
70
|
+
requirements:
|
71
|
+
- - "~>"
|
72
|
+
- !ruby/object:Gem::Version
|
73
|
+
version: '12.0'
|
74
|
+
type: :development
|
75
|
+
prerelease: false
|
76
|
+
version_requirements: !ruby/object:Gem::Requirement
|
77
|
+
requirements:
|
78
|
+
- - "~>"
|
79
|
+
- !ruby/object:Gem::Version
|
80
|
+
version: '12.0'
|
81
|
+
- !ruby/object:Gem::Dependency
|
82
|
+
name: redis
|
83
|
+
requirement: !ruby/object:Gem::Requirement
|
84
|
+
requirements:
|
85
|
+
- - "~>"
|
86
|
+
- !ruby/object:Gem::Version
|
87
|
+
version: '4.1'
|
88
|
+
type: :development
|
89
|
+
prerelease: false
|
90
|
+
version_requirements: !ruby/object:Gem::Requirement
|
91
|
+
requirements:
|
92
|
+
- - "~>"
|
93
|
+
- !ruby/object:Gem::Version
|
94
|
+
version: '4.1'
|
95
|
+
- !ruby/object:Gem::Dependency
|
96
|
+
name: pg
|
97
|
+
requirement: !ruby/object:Gem::Requirement
|
98
|
+
requirements:
|
99
|
+
- - "~>"
|
100
|
+
- !ruby/object:Gem::Version
|
101
|
+
version: '1.2'
|
102
|
+
type: :development
|
103
|
+
prerelease: false
|
104
|
+
version_requirements: !ruby/object:Gem::Requirement
|
105
|
+
requirements:
|
106
|
+
- - "~>"
|
107
|
+
- !ruby/object:Gem::Version
|
108
|
+
version: '1.2'
|
109
|
+
- !ruby/object:Gem::Dependency
|
110
|
+
name: rack
|
111
|
+
requirement: !ruby/object:Gem::Requirement
|
112
|
+
requirements:
|
113
|
+
- - ">="
|
114
|
+
- !ruby/object:Gem::Version
|
115
|
+
version: '0'
|
116
|
+
type: :development
|
117
|
+
prerelease: false
|
118
|
+
version_requirements: !ruby/object:Gem::Requirement
|
119
|
+
requirements:
|
120
|
+
- - ">="
|
121
|
+
- !ruby/object:Gem::Version
|
122
|
+
version: '0'
|
123
|
+
- !ruby/object:Gem::Dependency
|
124
|
+
name: rspec
|
125
|
+
requirement: !ruby/object:Gem::Requirement
|
126
|
+
requirements:
|
127
|
+
- - "~>"
|
128
|
+
- !ruby/object:Gem::Version
|
129
|
+
version: '3.0'
|
130
|
+
type: :development
|
131
|
+
prerelease: false
|
132
|
+
version_requirements: !ruby/object:Gem::Requirement
|
133
|
+
requirements:
|
134
|
+
- - "~>"
|
135
|
+
- !ruby/object:Gem::Version
|
136
|
+
version: '3.0'
|
137
|
+
- !ruby/object:Gem::Dependency
|
138
|
+
name: byebug
|
139
|
+
requirement: !ruby/object:Gem::Requirement
|
140
|
+
requirements:
|
141
|
+
- - ">="
|
142
|
+
- !ruby/object:Gem::Version
|
143
|
+
version: '0'
|
144
|
+
type: :development
|
145
|
+
prerelease: false
|
146
|
+
version_requirements: !ruby/object:Gem::Requirement
|
147
|
+
requirements:
|
148
|
+
- - ">="
|
149
|
+
- !ruby/object:Gem::Version
|
150
|
+
version: '0'
|
151
|
+
- !ruby/object:Gem::Dependency
|
152
|
+
name: rubocop-discourse
|
153
|
+
requirement: !ruby/object:Gem::Requirement
|
154
|
+
requirements:
|
155
|
+
- - ">="
|
156
|
+
- !ruby/object:Gem::Version
|
157
|
+
version: '0'
|
158
|
+
type: :development
|
159
|
+
prerelease: false
|
160
|
+
version_requirements: !ruby/object:Gem::Requirement
|
161
|
+
requirements:
|
162
|
+
- - ">="
|
163
|
+
- !ruby/object:Gem::Version
|
164
|
+
version: '0'
|
165
|
+
- !ruby/object:Gem::Dependency
|
166
|
+
name: syntax_tree
|
167
|
+
requirement: !ruby/object:Gem::Requirement
|
168
|
+
requirements:
|
169
|
+
- - ">="
|
170
|
+
- !ruby/object:Gem::Version
|
171
|
+
version: '0'
|
172
|
+
type: :development
|
173
|
+
prerelease: false
|
174
|
+
version_requirements: !ruby/object:Gem::Requirement
|
175
|
+
requirements:
|
176
|
+
- - ">="
|
177
|
+
- !ruby/object:Gem::Version
|
178
|
+
version: '0'
|
179
|
+
- !ruby/object:Gem::Dependency
|
180
|
+
name: syntax_tree-disable_ternary
|
181
|
+
requirement: !ruby/object:Gem::Requirement
|
182
|
+
requirements:
|
183
|
+
- - ">="
|
184
|
+
- !ruby/object:Gem::Version
|
185
|
+
version: '0'
|
186
|
+
type: :development
|
187
|
+
prerelease: false
|
188
|
+
version_requirements: !ruby/object:Gem::Requirement
|
189
|
+
requirements:
|
190
|
+
- - ">="
|
191
|
+
- !ruby/object:Gem::Version
|
192
|
+
version: '0'
|
67
193
|
description:
|
68
194
|
email:
|
69
195
|
- tgx@discourse.org
|
@@ -75,6 +201,7 @@ files:
|
|
75
201
|
- ".gitignore"
|
76
202
|
- ".rspec"
|
77
203
|
- ".rubocop.yml"
|
204
|
+
- ".streerc"
|
78
205
|
- CHANGELOG.md
|
79
206
|
- CODE_OF_CONDUCT.md
|
80
207
|
- Gemfile
|
@@ -110,7 +237,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
|
|
110
237
|
requirements:
|
111
238
|
- - ">="
|
112
239
|
- !ruby/object:Gem::Version
|
113
|
-
version: 2.
|
240
|
+
version: 2.7.0
|
114
241
|
required_rubygems_version: !ruby/object:Gem::Requirement
|
115
242
|
requirements:
|
116
243
|
- - ">="
|