rails_failover 0.2.0 → 0.5.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.rubocop.yml +4 -0
- data/CHANGELOG.md +12 -0
- data/Gemfile +6 -5
- data/Gemfile.lock +65 -2
- data/README.md +79 -10
- data/bin/console +2 -1
- data/bin/rspec +1 -1
- data/lib/rails_failover.rb +0 -2
- data/lib/rails_failover/active_record.rb +62 -0
- data/lib/rails_failover/active_record/handler.rb +140 -0
- data/lib/rails_failover/active_record/middleware.rb +103 -0
- data/lib/rails_failover/active_record/railtie.rb +58 -0
- data/lib/rails_failover/redis.rb +32 -15
- data/lib/rails_failover/redis/connector.rb +14 -9
- data/lib/rails_failover/redis/handler.rb +193 -0
- data/lib/rails_failover/version.rb +1 -1
- data/makefile +14 -24
- data/postgresql.mk +54 -0
- data/rails_failover.gemspec +4 -2
- data/redis.mk +28 -0
- metadata +29 -8
- data/lib/rails_failover/redis/failover_handler.rb +0 -109
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 2bff6dd429dd01de4684f7ec70f8ccaeb36f403be9c25f2dceb5789ea4d19bef
|
4
|
+
data.tar.gz: 0b7a7b8a1ad27498e49079c88604d5365800823a3c5c8b7740af7d71683f59a8
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 0013d14b3dd577feb3eb627a223ffa94f1b781310194d8a430524f4f1f4728ad32e117ffbd333ae476b1d87194c202b9e3511e8989d2aefa718f3c97f6270e2a
|
7
|
+
data.tar.gz: c6fbfb3a9a22557a73b1208f39c51bd876b607c50b22438183fd0249d92af4926ea6cb517dea3ce556bbb08c02dd8a6a4f93d6cd1c7699ebd8e3d54eecf955f2
|
data/.rubocop.yml
CHANGED
data/CHANGELOG.md
ADDED
@@ -0,0 +1,12 @@
|
|
1
|
+
# Changelog
|
2
|
+
All notable changes to this project will be documented in this file.
|
3
|
+
|
4
|
+
The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
|
5
|
+
and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
|
6
|
+
|
7
|
+
## [Unreleased]
|
8
|
+
|
9
|
+
## [0.5.2] - 2020-06-23
|
10
|
+
|
11
|
+
### Changed
|
12
|
+
- FIX: Only rescue from connection errors.q
|
data/Gemfile
CHANGED
@@ -6,8 +6,9 @@ gemspec
|
|
6
6
|
|
7
7
|
gem "rake", "~> 12.0"
|
8
8
|
gem "rspec", "~> 3.0"
|
9
|
-
|
10
|
-
|
11
|
-
|
12
|
-
|
13
|
-
|
9
|
+
gem 'rubocop-discourse'
|
10
|
+
gem 'byebug'
|
11
|
+
gem 'redis', '~> 4.1'
|
12
|
+
gem 'pg', '~> 1.2'
|
13
|
+
gem 'activerecord', '~> 6.0'
|
14
|
+
gem 'rack'
|
data/Gemfile.lock
CHANGED
@@ -1,18 +1,72 @@
|
|
1
1
|
PATH
|
2
2
|
remote: .
|
3
3
|
specs:
|
4
|
-
rails_failover (0.
|
5
|
-
|
4
|
+
rails_failover (0.5.2)
|
5
|
+
activerecord (~> 6.0)
|
6
|
+
railties (~> 6.0)
|
6
7
|
|
7
8
|
GEM
|
8
9
|
remote: https://rubygems.org/
|
9
10
|
specs:
|
11
|
+
actionpack (6.0.3.1)
|
12
|
+
actionview (= 6.0.3.1)
|
13
|
+
activesupport (= 6.0.3.1)
|
14
|
+
rack (~> 2.0, >= 2.0.8)
|
15
|
+
rack-test (>= 0.6.3)
|
16
|
+
rails-dom-testing (~> 2.0)
|
17
|
+
rails-html-sanitizer (~> 1.0, >= 1.2.0)
|
18
|
+
actionview (6.0.3.1)
|
19
|
+
activesupport (= 6.0.3.1)
|
20
|
+
builder (~> 3.1)
|
21
|
+
erubi (~> 1.4)
|
22
|
+
rails-dom-testing (~> 2.0)
|
23
|
+
rails-html-sanitizer (~> 1.1, >= 1.2.0)
|
24
|
+
activemodel (6.0.3.1)
|
25
|
+
activesupport (= 6.0.3.1)
|
26
|
+
activerecord (6.0.3.1)
|
27
|
+
activemodel (= 6.0.3.1)
|
28
|
+
activesupport (= 6.0.3.1)
|
29
|
+
activesupport (6.0.3.1)
|
30
|
+
concurrent-ruby (~> 1.0, >= 1.0.2)
|
31
|
+
i18n (>= 0.7, < 2)
|
32
|
+
minitest (~> 5.1)
|
33
|
+
tzinfo (~> 1.1)
|
34
|
+
zeitwerk (~> 2.2, >= 2.2.2)
|
10
35
|
ast (2.4.0)
|
36
|
+
builder (3.2.4)
|
11
37
|
byebug (11.1.3)
|
38
|
+
concurrent-ruby (1.1.6)
|
39
|
+
crass (1.0.6)
|
12
40
|
diff-lcs (1.3)
|
41
|
+
erubi (1.9.0)
|
42
|
+
i18n (1.8.2)
|
43
|
+
concurrent-ruby (~> 1.0)
|
44
|
+
loofah (2.6.0)
|
45
|
+
crass (~> 1.0.2)
|
46
|
+
nokogiri (>= 1.5.9)
|
47
|
+
method_source (1.0.0)
|
48
|
+
mini_portile2 (2.4.0)
|
49
|
+
minitest (5.14.1)
|
50
|
+
nokogiri (1.10.9)
|
51
|
+
mini_portile2 (~> 2.4.0)
|
13
52
|
parallel (1.19.1)
|
14
53
|
parser (2.7.1.2)
|
15
54
|
ast (~> 2.4.0)
|
55
|
+
pg (1.2.3)
|
56
|
+
rack (2.2.2)
|
57
|
+
rack-test (1.1.0)
|
58
|
+
rack (>= 1.0, < 3)
|
59
|
+
rails-dom-testing (2.0.3)
|
60
|
+
activesupport (>= 4.2.0)
|
61
|
+
nokogiri (>= 1.6)
|
62
|
+
rails-html-sanitizer (1.3.0)
|
63
|
+
loofah (~> 2.3)
|
64
|
+
railties (6.0.3.1)
|
65
|
+
actionpack (= 6.0.3.1)
|
66
|
+
activesupport (= 6.0.3.1)
|
67
|
+
method_source
|
68
|
+
rake (>= 0.8.7)
|
69
|
+
thor (>= 0.20.3, < 2.0)
|
16
70
|
rainbow (3.0.0)
|
17
71
|
rake (12.3.3)
|
18
72
|
redis (4.1.4)
|
@@ -43,15 +97,24 @@ GEM
|
|
43
97
|
rubocop-rspec (1.39.0)
|
44
98
|
rubocop (>= 0.68.1)
|
45
99
|
ruby-progressbar (1.10.1)
|
100
|
+
thor (1.0.1)
|
101
|
+
thread_safe (0.3.6)
|
102
|
+
tzinfo (1.2.7)
|
103
|
+
thread_safe (~> 0.1)
|
46
104
|
unicode-display_width (1.7.0)
|
105
|
+
zeitwerk (2.3.0)
|
47
106
|
|
48
107
|
PLATFORMS
|
49
108
|
ruby
|
50
109
|
|
51
110
|
DEPENDENCIES
|
111
|
+
activerecord (~> 6.0)
|
52
112
|
byebug
|
113
|
+
pg (~> 1.2)
|
114
|
+
rack
|
53
115
|
rails_failover!
|
54
116
|
rake (~> 12.0)
|
117
|
+
redis (~> 4.1)
|
55
118
|
rspec (~> 3.0)
|
56
119
|
rubocop-discourse
|
57
120
|
|
data/README.md
CHANGED
@@ -1,13 +1,16 @@
|
|
1
1
|
# RailsFailover
|
2
2
|
|
3
|
-
|
3
|
+
Automatic failover and recovery for primary/replica setup for:
|
4
|
+
|
5
|
+
1. Redis
|
6
|
+
1. ActiveRecord (PostgreSQL/MySQL)
|
4
7
|
|
5
8
|
## Installation
|
6
9
|
|
7
10
|
Add this line to your application's Gemfile:
|
8
11
|
|
9
12
|
```ruby
|
10
|
-
gem 'rails_failover'
|
13
|
+
gem 'rails_failover', require: false
|
11
14
|
```
|
12
15
|
|
13
16
|
And then execute:
|
@@ -20,35 +23,101 @@ Or install it yourself as:
|
|
20
23
|
|
21
24
|
## Usage
|
22
25
|
|
26
|
+
### ActiveRecord
|
27
|
+
|
28
|
+
In `config/application.rb` add `require 'rails_failover/active_record'` after `require "active_record/railtie"`.
|
29
|
+
|
30
|
+
In your database configuration, simply add `replica_host` and `replica_port` to your database configuration.
|
31
|
+
|
32
|
+
```
|
33
|
+
production:
|
34
|
+
host: <primary db server host>
|
35
|
+
port: <primary db server port>
|
36
|
+
replica_host: <replica db server host>
|
37
|
+
replica_port: <replica db server port>
|
38
|
+
```
|
39
|
+
|
40
|
+
The gem will automatically create an `ActiveRecord::ConnectionAdapters::ConnectionHandler` with the `ActiveRecord::Base.reading_role` as the `handler_key`.
|
41
|
+
|
42
|
+
#### Failover/Fallback Hooks
|
43
|
+
|
44
|
+
```
|
45
|
+
RailsFailover::ActiveRecord.on_failover do
|
46
|
+
# Enable readonly mode
|
47
|
+
end
|
48
|
+
|
49
|
+
RailsFailover::ActiveRecord.on_fallback do
|
50
|
+
# Disable readonly mode
|
51
|
+
end
|
52
|
+
```
|
53
|
+
|
54
|
+
#### Multiple connection handlers
|
55
|
+
|
56
|
+
Note: This API is unstable and is likely to changes when Rails 6.1 is released with sharding support.
|
57
|
+
|
58
|
+
```
|
59
|
+
# config/database.yml
|
60
|
+
|
61
|
+
production:
|
62
|
+
primary:
|
63
|
+
host: <primary db server host>
|
64
|
+
port: <primary db server port>
|
65
|
+
replica_host: <replica db server host>
|
66
|
+
replica_port: <replica db server port>
|
67
|
+
second_database_writing:
|
68
|
+
host: <primary db server host>
|
69
|
+
port: <primary db server port>
|
70
|
+
replica_host: <replica db server host>
|
71
|
+
replica_port: <replica db server port>
|
72
|
+
|
73
|
+
# In your ActiveRecord base model or model.
|
74
|
+
|
75
|
+
connects_to database: { writing: :primary, second_database_writing: :second_database_writing
|
76
|
+
```
|
77
|
+
|
23
78
|
### Redis
|
24
79
|
|
80
|
+
Add `require 'rails_failover/redis'` before creating a `Redis` instance.
|
81
|
+
|
25
82
|
```
|
26
83
|
Redis.new(host: "127.0.0.1", port: 6379, replica_host: "127.0.0.1", replica_port: 6380, connector: RailsFailover::Redis::Connector))
|
27
84
|
```
|
28
85
|
|
29
|
-
Callbacks can be registered when the
|
30
|
-
|
86
|
+
Callbacks can be registered when the primary connection is down and when it is up.
|
31
87
|
|
32
88
|
```
|
33
|
-
RailsFailover::Redis.
|
89
|
+
RailsFailover::Redis.on_failover_callback do
|
34
90
|
# Switch site to read-only mode
|
35
91
|
end
|
36
92
|
|
37
|
-
RailsFailover::Redis.
|
93
|
+
RailsFailover::Redis.on_fallback_callback do
|
38
94
|
# Switch site out of read-only mode
|
39
95
|
end
|
40
96
|
```
|
41
97
|
|
42
98
|
## Development
|
43
99
|
|
44
|
-
After checking out the repo, run `bin/setup` to install dependencies.
|
100
|
+
After checking out the repo, run `bin/setup` to install dependencies. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
|
45
101
|
|
46
102
|
To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release`, which will create a git tag for the version, push git commits and tags, and push the `.gem` file to [rubygems.org](https://rubygems.org).
|
47
103
|
|
48
|
-
|
104
|
+
### Testing
|
105
|
+
|
106
|
+
#### ActiveRecord
|
107
|
+
|
108
|
+
The ActiveRecord failover tests are run against a dummy Rails server. Run the following commands to run the test:
|
109
|
+
|
110
|
+
1. `make setup_pg`
|
111
|
+
1. `make start_pg`
|
112
|
+
1. `bin/rspec active_record`. You may also run the tests with more unicorn workers by adding the `UNICORN_WORKERS` env variable.
|
49
113
|
|
50
|
-
|
114
|
+
#### Redis
|
115
|
+
|
116
|
+
`bin/rspec redis`
|
117
|
+
|
118
|
+
## Contributing
|
51
119
|
|
120
|
+
Bug reports and pull requests are welcome on GitHub at https://github.com/discourse/rails_failover. This project is intended to be a safe, welcoming space for collaboration, and contributors are expected to adhere to the [code of conduct](https://github.com/discourse/rails_failover/blob/master/CODE_OF_CONDUCT.md).
|
52
121
|
|
53
122
|
## License
|
54
123
|
|
@@ -56,4 +125,4 @@ The gem is available as open source under the terms of the [MIT License](https:/
|
|
56
125
|
|
57
126
|
## Code of Conduct
|
58
127
|
|
59
|
-
Everyone interacting in the RailsFailover project's codebases, issue trackers, chat rooms and mailing lists is expected to follow the [code of conduct](https://github.com/
|
128
|
+
Everyone interacting in the RailsFailover project's codebases, issue trackers, chat rooms and mailing lists is expected to follow the [code of conduct](https://github.com/discourse/rails_failover/blob/master/CODE_OF_CONDUCT.md).
|
data/bin/console
CHANGED
@@ -3,7 +3,8 @@
|
|
3
3
|
|
4
4
|
require "bundler/setup"
|
5
5
|
require "rails_failover"
|
6
|
-
require
|
6
|
+
require "rails_failover/redis"
|
7
|
+
require "rails_failover/active_record"
|
7
8
|
|
8
9
|
# You can add fixtures and/or initialization code here to make experimenting
|
9
10
|
# with your gem easier. You can also use a different console, if you like.
|
data/bin/rspec
CHANGED
@@ -1,2 +1,2 @@
|
|
1
1
|
#!/usr/bin/env bash
|
2
|
-
make
|
2
|
+
RSPEC_PATH=$2 make -s $1
|
data/lib/rails_failover.rb
CHANGED
@@ -0,0 +1,62 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require 'active_record'
|
4
|
+
|
5
|
+
if defined?(::Rails)
|
6
|
+
require_relative 'active_record/railtie'
|
7
|
+
end
|
8
|
+
|
9
|
+
require_relative 'active_record/middleware'
|
10
|
+
require_relative 'active_record/handler'
|
11
|
+
|
12
|
+
module RailsFailover
|
13
|
+
module ActiveRecord
|
14
|
+
def self.logger=(logger)
|
15
|
+
@logger = logger
|
16
|
+
end
|
17
|
+
|
18
|
+
def self.logger
|
19
|
+
@logger || Rails.logger
|
20
|
+
end
|
21
|
+
|
22
|
+
def self.verify_primary_frequency_seconds=(seconds)
|
23
|
+
@verify_primary_frequency_seconds = seconds
|
24
|
+
end
|
25
|
+
|
26
|
+
def self.verify_primary_frequency_seconds
|
27
|
+
@verify_primary_frequency_seconds || 5
|
28
|
+
end
|
29
|
+
|
30
|
+
def self.establish_reading_connection(handler, connection_spec)
|
31
|
+
config = connection_spec.config
|
32
|
+
|
33
|
+
if config[:replica_host] && config[:replica_port]
|
34
|
+
replica_config = config.dup
|
35
|
+
replica_config[:host] = replica_config.delete(:replica_host)
|
36
|
+
replica_config[:port] = replica_config.delete(:replica_port)
|
37
|
+
replica_config[:replica] = true
|
38
|
+
handler.establish_connection(replica_config)
|
39
|
+
end
|
40
|
+
end
|
41
|
+
|
42
|
+
def self.register_force_reading_role_callback(&block)
|
43
|
+
Middleware.force_reading_role_callback = block
|
44
|
+
end
|
45
|
+
|
46
|
+
def self.on_failover(&block)
|
47
|
+
@on_failover_callback = block
|
48
|
+
end
|
49
|
+
|
50
|
+
def self.on_failover_callback
|
51
|
+
@on_failover_callback
|
52
|
+
end
|
53
|
+
|
54
|
+
def self.on_fallback(&block)
|
55
|
+
@on_fallback_callback = block
|
56
|
+
end
|
57
|
+
|
58
|
+
def self.on_fallback_callback
|
59
|
+
@on_fallback_callback
|
60
|
+
end
|
61
|
+
end
|
62
|
+
end
|
@@ -0,0 +1,140 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
require 'singleton'
|
3
|
+
require 'monitor'
|
4
|
+
|
5
|
+
module RailsFailover
|
6
|
+
module ActiveRecord
|
7
|
+
class Handler
|
8
|
+
include Singleton
|
9
|
+
include MonitorMixin
|
10
|
+
|
11
|
+
VERIFY_FREQUENCY_BUFFER_PRECENT = 20
|
12
|
+
|
13
|
+
def initialize
|
14
|
+
@primaries_down = {}
|
15
|
+
@ancestor_pid = Process.pid
|
16
|
+
|
17
|
+
super() # Monitor#initialize
|
18
|
+
end
|
19
|
+
|
20
|
+
def verify_primary(handler_key)
|
21
|
+
mon_synchronize do
|
22
|
+
primary_down(handler_key)
|
23
|
+
return if @thread&.alive?
|
24
|
+
|
25
|
+
logger.warn "Failover for ActiveRecord has been initiated"
|
26
|
+
|
27
|
+
begin
|
28
|
+
RailsFailover::ActiveRecord.on_failover_callback&.call
|
29
|
+
rescue => e
|
30
|
+
logger.warn("RailsFailover::ActiveRecord.on_failover_callback failed: #{e.class} #{e.message}\n#{e.backtrace.join("\n")}")
|
31
|
+
end
|
32
|
+
|
33
|
+
@thread = Thread.new do
|
34
|
+
loop do
|
35
|
+
initiate_fallback_to_primary
|
36
|
+
|
37
|
+
if all_primaries_up
|
38
|
+
logger.warn "Fallback to primary for ActiveRecord has been completed."
|
39
|
+
|
40
|
+
begin
|
41
|
+
RailsFailover::ActiveRecord.on_fallback_callback&.call
|
42
|
+
rescue => e
|
43
|
+
logger.warn("RailsFailover::ActiveRecord.on_fallback_callback failed: #{e.class} #{e.message}\n#{e.backtrace.join("\n")}")
|
44
|
+
end
|
45
|
+
|
46
|
+
break
|
47
|
+
end
|
48
|
+
end
|
49
|
+
end
|
50
|
+
end
|
51
|
+
end
|
52
|
+
|
53
|
+
def initiate_fallback_to_primary
|
54
|
+
frequency = RailsFailover::ActiveRecord.verify_primary_frequency_seconds
|
55
|
+
sleep(frequency * ((rand(VERIFY_FREQUENCY_BUFFER_PRECENT) + 100) / 100.0))
|
56
|
+
|
57
|
+
active_handler_keys = []
|
58
|
+
|
59
|
+
primaries_down.keys.each do |handler_key|
|
60
|
+
connection_handler = ::ActiveRecord::Base.connection_handlers[handler_key]
|
61
|
+
spec = connection_handler.retrieve_connection_pool(spec_name).spec
|
62
|
+
config = spec.config
|
63
|
+
logger.debug "#{Process.pid} Checking server for '#{handler_key} #{spec_name}'..."
|
64
|
+
connection_active = false
|
65
|
+
|
66
|
+
begin
|
67
|
+
connection = ::ActiveRecord::Base.public_send(spec.adapter_method, config)
|
68
|
+
connection_active = connection.active?
|
69
|
+
rescue => e
|
70
|
+
logger.debug "#{Process.pid} Connection to server for '#{handler_key} #{spec_name}' failed with '#{e.message}'"
|
71
|
+
ensure
|
72
|
+
connection.disconnect! if connection
|
73
|
+
end
|
74
|
+
|
75
|
+
if connection_active
|
76
|
+
logger.debug "#{Process.pid} Server for '#{handler_key} #{spec_name}' is active."
|
77
|
+
active_handler_keys << handler_key
|
78
|
+
end
|
79
|
+
end
|
80
|
+
|
81
|
+
active_handler_keys.each do |handler_key|
|
82
|
+
primary_up(handler_key)
|
83
|
+
end
|
84
|
+
end
|
85
|
+
|
86
|
+
def primary_down?(handler_key)
|
87
|
+
primaries_down[handler_key]
|
88
|
+
end
|
89
|
+
|
90
|
+
private
|
91
|
+
|
92
|
+
def all_primaries_up
|
93
|
+
mon_synchronize do
|
94
|
+
primaries_down.empty?
|
95
|
+
end
|
96
|
+
end
|
97
|
+
|
98
|
+
def primary_down(handler_key)
|
99
|
+
mon_synchronize do
|
100
|
+
primaries_down[handler_key] = true
|
101
|
+
end
|
102
|
+
end
|
103
|
+
|
104
|
+
def primary_up(handler_key)
|
105
|
+
mon_synchronize do
|
106
|
+
primaries_down.delete(handler_key)
|
107
|
+
end
|
108
|
+
end
|
109
|
+
|
110
|
+
def spec_name
|
111
|
+
::ActiveRecord::Base.connection_specification_name
|
112
|
+
end
|
113
|
+
|
114
|
+
def primaries_down
|
115
|
+
process_pid = Process.pid
|
116
|
+
return @primaries_down[process_pid] if @primaries_down[process_pid]
|
117
|
+
|
118
|
+
mon_synchronize do
|
119
|
+
if !@primaries_down[process_pid]
|
120
|
+
@primaries_down[process_pid] = @primaries_down[@ancestor_pid] || {}
|
121
|
+
|
122
|
+
if process_pid != @ancestor_pid
|
123
|
+
@primaries_down.delete(@ancestor_pid)
|
124
|
+
|
125
|
+
@primaries_down[process_pid].each_key do |handler_key|
|
126
|
+
verify_primary(handler_key)
|
127
|
+
end
|
128
|
+
end
|
129
|
+
end
|
130
|
+
|
131
|
+
@primaries_down[process_pid]
|
132
|
+
end
|
133
|
+
end
|
134
|
+
|
135
|
+
def logger
|
136
|
+
::Rails.logger
|
137
|
+
end
|
138
|
+
end
|
139
|
+
end
|
140
|
+
end
|