rails_failover 0.2.0 → 0.5.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 6b47ce7eef70576e83fe137949d50e280677f6f5e2444790a9bdf7cc1708f540
-  data.tar.gz: eea78a86a311a24926c83ab4450eae6ed59b0f39a60faafdf9affe050074aa7d
+  metadata.gz: 2bff6dd429dd01de4684f7ec70f8ccaeb36f403be9c25f2dceb5789ea4d19bef
+  data.tar.gz: 0b7a7b8a1ad27498e49079c88604d5365800823a3c5c8b7740af7d71683f59a8
 SHA512:
-  metadata.gz: 46aea75592369b8a0af7a48ee1898d2fdfecb8a80ebc777ba386052709678258d719a7e2e2d4d00f1c6c3ca7d158747eecb0a329373ff3a8f6423c6f70824f27
-  data.tar.gz: aa006f95128b6ec9fe600652c62d20c32b948aeff049cf2155d5366b97a57d851631e8de6c128f2f0587c0e4d97a797252b5c2f196a98d7d245e3b0a539d8b53
+  metadata.gz: 0013d14b3dd577feb3eb627a223ffa94f1b781310194d8a430524f4f1f4728ad32e117ffbd333ae476b1d87194c202b9e3511e8989d2aefa718f3c97f6270e2a
+  data.tar.gz: c6fbfb3a9a22557a73b1208f39c51bd876b607c50b22438183fd0249d92af4926ea6cb517dea3ce556bbb08c02dd8a6a4f93d6cd1c7699ebd8e3d54eecf955f2

data/.rubocop.yml

@@ -1,3 +1,7 @@
+AllCops:
+  Exclude:
+    - spec/support/dummy_app/**/*
+
 inherit_gem:
   rubocop-discourse: default.yml
 

data/CHANGELOG.md

@@ -0,0 +1,12 @@
+# Changelog
+All notable changes to this project will be documented in this file.
+
+The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
+and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
+
+## [Unreleased]
+
+## [0.5.2] - 2020-06-23
+
+### Changed
+- FIX: Only rescue from connection errors.q

data/Gemfile

@@ -6,8 +6,9 @@ gemspec
 
 gem "rake", "~> 12.0"
 gem "rspec", "~> 3.0"
-
-group :development do
-  gem 'rubocop-discourse'
-  gem 'byebug'
-end
+gem 'rubocop-discourse'
+gem 'byebug'
+gem 'redis', '~> 4.1'
+gem 'pg', '~> 1.2'
+gem 'activerecord', '~> 6.0'
+gem 'rack'

data/Gemfile.lock

@@ -1,18 +1,72 @@
 PATH
   remote: .
   specs:
-    rails_failover (0.1.0)
-      redis (~> 4)
+    rails_failover (0.5.2)
+      activerecord (~> 6.0)
+      railties (~> 6.0)
 
 GEM
   remote: https://rubygems.org/
   specs:
+    actionpack (6.0.3.1)
+      actionview (= 6.0.3.1)
+      activesupport (= 6.0.3.1)
+      rack (~> 2.0, >= 2.0.8)
+      rack-test (>= 0.6.3)
+      rails-dom-testing (~> 2.0)
+      rails-html-sanitizer (~> 1.0, >= 1.2.0)
+    actionview (6.0.3.1)
+      activesupport (= 6.0.3.1)
+      builder (~> 3.1)
+      erubi (~> 1.4)
+      rails-dom-testing (~> 2.0)
+      rails-html-sanitizer (~> 1.1, >= 1.2.0)
+    activemodel (6.0.3.1)
+      activesupport (= 6.0.3.1)
+    activerecord (6.0.3.1)
+      activemodel (= 6.0.3.1)
+      activesupport (= 6.0.3.1)
+    activesupport (6.0.3.1)
+      concurrent-ruby (~> 1.0, >= 1.0.2)
+      i18n (>= 0.7, < 2)
+      minitest (~> 5.1)
+      tzinfo (~> 1.1)
+      zeitwerk (~> 2.2, >= 2.2.2)
     ast (2.4.0)
+    builder (3.2.4)
     byebug (11.1.3)
+    concurrent-ruby (1.1.6)
+    crass (1.0.6)
     diff-lcs (1.3)
+    erubi (1.9.0)
+    i18n (1.8.2)
+      concurrent-ruby (~> 1.0)
+    loofah (2.6.0)
+      crass (~> 1.0.2)
+      nokogiri (>= 1.5.9)
+    method_source (1.0.0)
+    mini_portile2 (2.4.0)
+    minitest (5.14.1)
+    nokogiri (1.10.9)
+      mini_portile2 (~> 2.4.0)
     parallel (1.19.1)
     parser (2.7.1.2)
       ast (~> 2.4.0)
+    pg (1.2.3)
+    rack (2.2.2)
+    rack-test (1.1.0)
+      rack (>= 1.0, < 3)
+    rails-dom-testing (2.0.3)
+      activesupport (>= 4.2.0)
+      nokogiri (>= 1.6)
+    rails-html-sanitizer (1.3.0)
+      loofah (~> 2.3)
+    railties (6.0.3.1)
+      actionpack (= 6.0.3.1)
+      activesupport (= 6.0.3.1)
+      method_source
+      rake (>= 0.8.7)
+      thor (>= 0.20.3, < 2.0)
     rainbow (3.0.0)
     rake (12.3.3)
     redis (4.1.4)
@@ -43,15 +97,24 @@ GEM
     rubocop-rspec (1.39.0)
       rubocop (>= 0.68.1)
     ruby-progressbar (1.10.1)
+    thor (1.0.1)
+    thread_safe (0.3.6)
+    tzinfo (1.2.7)
+      thread_safe (~> 0.1)
     unicode-display_width (1.7.0)
+    zeitwerk (2.3.0)
 
 PLATFORMS
   ruby
 
 DEPENDENCIES
+  activerecord (~> 6.0)
   byebug
+  pg (~> 1.2)
+  rack
   rails_failover!
   rake (~> 12.0)
+  redis (~> 4.1)
   rspec (~> 3.0)
   rubocop-discourse
 

data/README.md

@@ -1,13 +1,16 @@
 # RailsFailover
 
-* Automatic failover and recovery for simple master-replica Redis setup
+Automatic failover and recovery for primary/replica setup for:
+
+1. Redis
+1. ActiveRecord (PostgreSQL/MySQL)
 
 ## Installation
 
 Add this line to your application's Gemfile:
 
 ```ruby
-gem 'rails_failover'
+gem 'rails_failover', require: false
 ```
 
 And then execute:
@@ -20,35 +23,101 @@ Or install it yourself as:
 
 ## Usage
 
+### ActiveRecord
+
+In `config/application.rb` add `require 'rails_failover/active_record'` after `require "active_record/railtie"`.
+
+In your database configuration, simply add `replica_host` and `replica_port` to your database configuration.
+
+```
+production:
+  host: <primary db server host>
+  port: <primary db server port>
+  replica_host: <replica db server host>
+  replica_port: <replica db server port>
+```
+
+The gem will automatically create an `ActiveRecord::ConnectionAdapters::ConnectionHandler` with the `ActiveRecord::Base.reading_role` as the `handler_key`.
+
+#### Failover/Fallback Hooks
+
+```
+RailsFailover::ActiveRecord.on_failover do
+  # Enable readonly mode
+end
+
+RailsFailover::ActiveRecord.on_fallback do
+  # Disable readonly mode
+end
+```
+
+#### Multiple connection handlers
+
+Note: This API is unstable and is likely to changes when Rails 6.1 is released with sharding support.
+
+```
+# config/database.yml
+
+production:
+  primary:
+    host: <primary db server host>
+    port: <primary db server port>
+    replica_host: <replica db server host>
+    replica_port: <replica db server port>
+  second_database_writing:
+    host: <primary db server host>
+    port: <primary db server port>
+    replica_host: <replica db server host>
+    replica_port: <replica db server port>
+
+# In your ActiveRecord base model or model.
+
+connects_to database: { writing: :primary, second_database_writing: :second_database_writing
+```
+
 ### Redis
 
+Add `require 'rails_failover/redis'` before creating a `Redis` instance.
+
 ```
 Redis.new(host: "127.0.0.1", port: 6379, replica_host: "127.0.0.1", replica_port: 6380, connector: RailsFailover::Redis::Connector))
 ```
 
-Callbacks can be registered when the master connection is down and when it is up.
-
+Callbacks can be registered when the primary connection is down and when it is up.
 
 ```
-RailsFailover::Redis.register_master_up_callback do
+RailsFailover::Redis.on_failover_callback do
   # Switch site to read-only mode
 end
 
-RailsFailover::Redis.register_master_down_callback do
+RailsFailover::Redis.on_fallback_callback do
   # Switch site out of read-only mode
 end
 ```
 
 ## Development
 
-After checking out the repo, run `bin/setup` to install dependencies. Then, run `bin/rspec` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
+After checking out the repo, run `bin/setup` to install dependencies. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
 
 To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release`, which will create a git tag for the version, push git commits and tags, and push the `.gem` file to [rubygems.org](https://rubygems.org).
 
-## Contributing
+### Testing
+
+#### ActiveRecord
+
+The ActiveRecord failover tests are run against a dummy Rails server. Run the following commands to run the test:
+
+1. `make setup_pg`
+1. `make start_pg`
+1. `bin/rspec active_record`. You may also run the tests with more unicorn workers by adding the `UNICORN_WORKERS` env variable.
 
-Bug reports and pull requests are welcome on GitHub at https://github.com/[USERNAME]/rails_failover. This project is intended to be a safe, welcoming space for collaboration, and contributors are expected to adhere to the [code of conduct](https://github.com/[USERNAME]/rails_failover/blob/master/CODE_OF_CONDUCT.md).
+#### Redis
+
+`bin/rspec redis`
+
+## Contributing
 
+Bug reports and pull requests are welcome on GitHub at https://github.com/discourse/rails_failover. This project is intended to be a safe, welcoming space for collaboration, and contributors are expected to adhere to the [code of conduct](https://github.com/discourse/rails_failover/blob/master/CODE_OF_CONDUCT.md).
 
 ## License
 
@@ -56,4 +125,4 @@ The gem is available as open source under the terms of the [MIT License](https:/
 
 ## Code of Conduct
 
-Everyone interacting in the RailsFailover project's codebases, issue trackers, chat rooms and mailing lists is expected to follow the [code of conduct](https://github.com/[USERNAME]/rails_failover/blob/master/CODE_OF_CONDUCT.md).
+Everyone interacting in the RailsFailover project's codebases, issue trackers, chat rooms and mailing lists is expected to follow the [code of conduct](https://github.com/discourse/rails_failover/blob/master/CODE_OF_CONDUCT.md).

data/bin/console

@@ -3,7 +3,8 @@
 
 require "bundler/setup"
 require "rails_failover"
-require 'redis'
+require "rails_failover/redis"
+require "rails_failover/active_record"
 
 # You can add fixtures and/or initialization code here to make experimenting
 # with your gem easier. You can also use a different console, if you like.

data/bin/rspec

@@ -1,2 +1,2 @@
 #!/usr/bin/env bash
-make RSPEC_PATH=$1
+RSPEC_PATH=$2 make -s $1

data/lib/rails_failover.rb

@@ -1,8 +1,6 @@
 # frozen_string_literal: true
 
 require "rails_failover/version"
-require "redis/patches/client"
-require "rails_failover/redis"
 
 module RailsFailover
   class Error < StandardError; end

data/lib/rails_failover/active_record.rb

@@ -0,0 +1,62 @@
+# frozen_string_literal: true
+
+require 'active_record'
+
+if defined?(::Rails)
+  require_relative 'active_record/railtie'
+end
+
+require_relative 'active_record/middleware'
+require_relative 'active_record/handler'
+
+module RailsFailover
+  module ActiveRecord
+    def self.logger=(logger)
+      @logger = logger
+    end
+
+    def self.logger
+      @logger || Rails.logger
+    end
+
+    def self.verify_primary_frequency_seconds=(seconds)
+      @verify_primary_frequency_seconds = seconds
+    end
+
+    def self.verify_primary_frequency_seconds
+      @verify_primary_frequency_seconds || 5
+    end
+
+    def self.establish_reading_connection(handler, connection_spec)
+      config = connection_spec.config
+
+      if config[:replica_host] && config[:replica_port]
+        replica_config = config.dup
+        replica_config[:host] = replica_config.delete(:replica_host)
+        replica_config[:port] = replica_config.delete(:replica_port)
+        replica_config[:replica] = true
+        handler.establish_connection(replica_config)
+      end
+    end
+
+    def self.register_force_reading_role_callback(&block)
+      Middleware.force_reading_role_callback = block
+    end
+
+    def self.on_failover(&block)
+      @on_failover_callback = block
+    end
+
+    def self.on_failover_callback
+      @on_failover_callback
+    end
+
+    def self.on_fallback(&block)
+      @on_fallback_callback = block
+    end
+
+    def self.on_fallback_callback
+      @on_fallback_callback
+    end
+  end
+end

data/lib/rails_failover/active_record/handler.rb

@@ -0,0 +1,140 @@
+# frozen_string_literal: true
+require 'singleton'
+require 'monitor'
+
+module RailsFailover
+  module ActiveRecord
+    class Handler
+      include Singleton
+      include MonitorMixin
+
+      VERIFY_FREQUENCY_BUFFER_PRECENT = 20
+
+      def initialize
+        @primaries_down = {}
+        @ancestor_pid = Process.pid
+
+        super() # Monitor#initialize
+      end
+
+      def verify_primary(handler_key)
+        mon_synchronize do
+          primary_down(handler_key)
+          return if @thread&.alive?
+
+          logger.warn "Failover for ActiveRecord has been initiated"
+
+          begin
+            RailsFailover::ActiveRecord.on_failover_callback&.call
+          rescue => e
+            logger.warn("RailsFailover::ActiveRecord.on_failover_callback failed: #{e.class} #{e.message}\n#{e.backtrace.join("\n")}")
+          end
+
+          @thread = Thread.new do
+            loop do
+              initiate_fallback_to_primary
+
+              if all_primaries_up
+                logger.warn "Fallback to primary for ActiveRecord has been completed."
+
+                begin
+                  RailsFailover::ActiveRecord.on_fallback_callback&.call
+                rescue => e
+                  logger.warn("RailsFailover::ActiveRecord.on_fallback_callback failed: #{e.class} #{e.message}\n#{e.backtrace.join("\n")}")
+                end
+
+                break
+              end
+            end
+          end
+        end
+      end
+
+      def initiate_fallback_to_primary
+        frequency = RailsFailover::ActiveRecord.verify_primary_frequency_seconds
+        sleep(frequency * ((rand(VERIFY_FREQUENCY_BUFFER_PRECENT) + 100) / 100.0))
+
+        active_handler_keys = []
+
+        primaries_down.keys.each do |handler_key|
+          connection_handler = ::ActiveRecord::Base.connection_handlers[handler_key]
+          spec = connection_handler.retrieve_connection_pool(spec_name).spec
+          config = spec.config
+          logger.debug "#{Process.pid} Checking server for '#{handler_key} #{spec_name}'..."
+          connection_active = false
+
+          begin
+            connection = ::ActiveRecord::Base.public_send(spec.adapter_method, config)
+            connection_active = connection.active?
+          rescue => e
+            logger.debug "#{Process.pid} Connection to server for '#{handler_key} #{spec_name}' failed with '#{e.message}'"
+          ensure
+            connection.disconnect! if connection
+          end
+
+          if connection_active
+            logger.debug "#{Process.pid} Server for '#{handler_key} #{spec_name}' is active."
+            active_handler_keys << handler_key
+          end
+        end
+
+        active_handler_keys.each do |handler_key|
+          primary_up(handler_key)
+        end
+      end
+
+      def primary_down?(handler_key)
+        primaries_down[handler_key]
+      end
+
+      private
+
+      def all_primaries_up
+        mon_synchronize do
+          primaries_down.empty?
+        end
+      end
+
+      def primary_down(handler_key)
+        mon_synchronize do
+          primaries_down[handler_key] = true
+        end
+      end
+
+      def primary_up(handler_key)
+        mon_synchronize do
+          primaries_down.delete(handler_key)
+        end
+      end
+
+      def spec_name
+        ::ActiveRecord::Base.connection_specification_name
+      end
+
+      def primaries_down
+        process_pid = Process.pid
+        return @primaries_down[process_pid] if @primaries_down[process_pid]
+
+        mon_synchronize do
+          if !@primaries_down[process_pid]
+            @primaries_down[process_pid] = @primaries_down[@ancestor_pid] || {}
+
+            if process_pid != @ancestor_pid
+              @primaries_down.delete(@ancestor_pid)
+
+              @primaries_down[process_pid].each_key do |handler_key|
+                verify_primary(handler_key)
+              end
+            end
+          end
+
+          @primaries_down[process_pid]
+        end
+      end
+
+      def logger
+        ::Rails.logger
+      end
+    end
+  end
+end