distribute_reads 0.2.4 → 0.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 5e39298a87b7d9af49c9bb2dd6bffc0174a569d994432fc86c6668f00d7b8b4b
-  data.tar.gz: 585c11f47cb258259b91ff9b3f7917ff83d06b7b1a7010732174790adac45917
+  metadata.gz: 4fb973f4af2e8828fc62a2c921605d595ee89bf521ba9f2b65f4c4f695225a76
+  data.tar.gz: 26ac05c9a814401ed03e56dfd7ceafa637e3ac0dcdb8aba79e12f9d664b5bbac
 SHA512:
-  metadata.gz: 527754102963062a8988612b29f92d2f6bf7f1d4e8084536983c7bf6fdbdaf92db5ae17e4d5e6ee7ae92865b5ad4e98a13a9710675dec85d8365acfeae4e7e72
-  data.tar.gz: 81c561439791aab9164e62d928d2a3ea60d9bbf9deb6217938fd36632d842e138106c11aacfb951ee48435a02bb0503d542ff861b736f9fcdac7ebfebdb5d6f2
+  metadata.gz: f40892ea166bb86ea89809d7b0e1cb74616ba55251f3208272319dab0d71322eb2c4efca3aaa2c134a9e0ae6adc1c20ea2cae16451d3c2fd6205e4f344cd27a8
+  data.tar.gz: 415fa474c9e3185f9d39b5410747f3d0a9feaf775e9a7e92d1595bc28d94e8ce69caa06d1c36eb646a98c94f963bc16ae27e54780260172689792066cf06ada2

data/CHANGELOG.md

@@ -1,3 +1,9 @@
+## 0.3.0
+
+- Use logger instead of stderr
+- Handle `NULL` replication lag for MySQL
+- Fixed replication lag check running on primary when replicas blacklisted
+
 ## 0.2.4
 
 - Added support for Aurora MySQL replication lag

data/LICENSE.txt

@@ -1,4 +1,4 @@
-Copyright (c) 2017 Andrew Kane
+Copyright (c) 2017-2019 Andrew Kane
 
 MIT License
 

data/README.md

@@ -137,6 +137,16 @@ DistributeReads.default_options = {
 }
 ```
 
+### Logging
+
+Messages about failover are logged to the Active Record logger by default. Set a different logger with:
+
+```ruby
+DistributeReads.logger = Logger.new(STDERR)
+```
+
+Or use `nil` to disable logging.
+
 ## Distribute Reads by Default
 
 At some point, you may wish to distribute reads by default.
@@ -161,6 +171,26 @@ Get replication lag in seconds
 DistributeReads.replication_lag
 ```
 
+Most of the time, Makara does a great job automatically routing queries to replicas. If it incorrectly routes a query to primary, you can use:
+
+```ruby
+distribute_reads(replica: true) do
+  # send all queries in block to replica
+end
+```
+
+## Rails 6
+
+Rails 6 has [native support for replicas](https://edgeguides.rubyonrails.org/active_record_multiple_databases.html) :tada:
+
+```ruby
+ActiveRecord::Base.connected_to(role: :reading) do
+  # do reads
+end
+```
+
+However, it’s not able to automatically route queries like Makara just yet.
+
 ## Thanks
 
 Thanks to [TaskRabbit](https://github.com/taskrabbit) for Makara, [Sherin Kurian](https://github.com/sherinkurian) for the max lag option, and [Nick Elser](https://github.com/nickelser) for the write-through cache.

data/lib/distribute_reads.rb

@@ -1,4 +1,8 @@
+# dependencies
+require "active_support"
 require "makara"
+
+# modules
 require "distribute_reads/appropriate_pool"
 require "distribute_reads/cache_store"
 require "distribute_reads/global_methods"
@@ -12,6 +16,7 @@ module DistributeReads
   class << self
     attr_accessor :by_default
     attr_accessor :default_options
+    attr_writer :logger
   end
   self.by_default = false
   self.default_options = {
@@ -19,15 +24,14 @@ module DistributeReads
     lag_failover: false
   }
 
-  def self.replication_lag(connection: nil)
-    distribute_reads do
-      lag(connection: connection)
+  def self.logger
+    unless defined?(@logger)
+      @logger = ActiveRecord::Base.logger
     end
+    @logger
   end
 
-  def self.lag(connection: nil)
-    raise DistributeReads::Error, "Don't use outside distribute_reads" unless Thread.current[:distribute_reads]
-
+  def self.replication_lag(connection: nil)
     connection ||= ActiveRecord::Base.connection
 
     replica_pool = connection.instance_variable_get(:@slave_pool)
@@ -35,35 +39,33 @@ module DistributeReads
       log "Multiple replicas available, lag only reported for one"
     end
 
-    if %w(PostgreSQL PostGIS).include?(connection.adapter_name)
-      # cache the version number
-      @server_version_num ||= {}
-      cache_key = connection.pool.object_id
-      @server_version_num[cache_key] ||= connection.execute("SHOW server_version_num").first["server_version_num"].to_i
-
-      lag_condition =
-        if @server_version_num[cache_key] >= 100000
-          "pg_last_wal_receive_lsn() = pg_last_wal_replay_lsn()"
-        else
-          "pg_last_xlog_receive_location() = pg_last_xlog_replay_location()"
-        end
-
-      connection.execute(
-        "SELECT CASE
-          WHEN NOT pg_is_in_recovery() OR #{lag_condition} THEN 0
-          ELSE EXTRACT (EPOCH FROM NOW() - pg_last_xact_replay_timestamp())
-        END AS lag".squish
-      ).first["lag"].to_f
-    elsif %w(MySQL Mysql2 Mysql2Spatial Mysql2Rgeo).include?(connection.adapter_name)
-      replica_value = Thread.current[:distribute_reads][:replica]
-      begin
-        # makara doesn't send SHOW queries to replica, so we must force it
-        Thread.current[:distribute_reads][:replica] = true
-
+    with_replica do
+      case connection.adapter_name
+      when "PostgreSQL", "PostGIS"
+        # cache the version number
+        @server_version_num ||= {}
+        cache_key = connection.pool.object_id
+        @server_version_num[cache_key] ||= connection.execute("SHOW server_version_num").first["server_version_num"].to_i
+
+        lag_condition =
+          if @server_version_num[cache_key] >= 100000
+            "pg_last_wal_receive_lsn() = pg_last_wal_replay_lsn()"
+          else
+            "pg_last_xlog_receive_location() = pg_last_xlog_replay_location()"
+          end
+
+        connection.execute(
+          "SELECT CASE
+            WHEN NOT pg_is_in_recovery() OR #{lag_condition} THEN 0
+            ELSE EXTRACT (EPOCH FROM NOW() - pg_last_xact_replay_timestamp())
+          END AS lag".squish
+        ).first["lag"].to_f
+      when "MySQL", "Mysql2", "Mysql2Spatial", "Mysql2Rgeo"
         @aurora_mysql ||= {}
         cache_key = connection.pool.object_id
 
         unless @aurora_mysql.key?(cache_key)
+          # makara doesn't send SHOW queries to replica by default
           @aurora_mysql[cache_key] = connection.exec_query("SHOW VARIABLES LIKE 'aurora_version'").to_hash.any?
         end
 
@@ -72,18 +74,41 @@ module DistributeReads
           status ? status["Replica_lag_in_msec"].to_f / 1000.0 : 0.0
         else
           status = connection.exec_query("SHOW SLAVE STATUS").to_hash.first
-          status ? status["Seconds_Behind_Master"].to_f : 0.0
+          if status
+            if status["Seconds_Behind_Master"].nil?
+              # replication stopped
+              # https://dev.mysql.com/doc/refman/8.0/en/show-slave-status.html
+              nil
+            else
+              status["Seconds_Behind_Master"].to_f
+            end
+          else
+            # not a replica
+            0.0
+          end
         end
-      ensure
-        Thread.current[:distribute_reads][:replica] = replica_value
+      when "SQLite"
+        # never a replica
+        0.0
+      else
+        raise DistributeReads::Error, "Option not supported with this adapter"
       end
-    else
-      raise DistributeReads::Error, "Option not supported with this adapter"
     end
   end
 
   def self.log(message)
-    warn "[distribute_reads] #{message}"
+    logger.info("[distribute_reads] #{message}") if logger
+  end
+
+  # private
+  def self.with_replica
+    previous_value = Thread.current[:distribute_reads]
+    begin
+      Thread.current[:distribute_reads] = {replica: true, failover: false}
+      yield
+    ensure
+      Thread.current[:distribute_reads] = previous_value
+    end
   end
 
   # private
@@ -105,8 +130,8 @@ module DistributeReads
   end
 end
 
-Makara::Proxy.send :prepend, DistributeReads::AppropriatePool
-Object.send :include, DistributeReads::GlobalMethods
+Makara::Proxy.prepend DistributeReads::AppropriatePool
+Object.include DistributeReads::GlobalMethods
 
 ActiveSupport.on_load(:active_job) do
   require "distribute_reads/job_methods"

data/lib/distribute_reads/global_methods.rb

@@ -20,8 +20,25 @@ module DistributeReads
         max_lag = options[:max_lag]
         if max_lag && !options[:primary]
           Array(options[:lag_on] || [ActiveRecord::Base]).each do |base_model|
-            if DistributeReads.lag(connection: base_model.connection) > max_lag
-              message = "Replica lag over #{max_lag} seconds#{options[:lag_on] ? " on #{base_model.name} connection" : ""}"
+            current_lag =
+              begin
+                DistributeReads.replication_lag(connection: base_model.connection)
+              rescue DistributeReads::NoReplicasAvailable => e
+                # TODO rescue more exceptions?
+                false
+              end
+
+            if !current_lag || current_lag > max_lag
+              message =
+                if current_lag.nil?
+                  "Replication stopped"
+                elsif !current_lag
+                  "No replicas available for lag check"
+                else
+                  "Replica lag over #{max_lag} seconds"
+                end
+
+              message = "#{message} on #{base_model.name} connection" if options[:lag_on]
 
               if options[:lag_failover]
                 # TODO possibly per connection

data/lib/distribute_reads/version.rb

@@ -1,3 +1,3 @@
 module DistributeReads
-  VERSION = "0.2.4"
+  VERSION = "0.3.0"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: distribute_reads
 version: !ruby/object:Gem::Version
-  version: 0.2.4
+  version: 0.3.0
 platform: ruby
 authors:
 - Andrew Kane
 autorequire: 
-bindir: exe
+bindir: bin
 cert_chain: []
-date: 2018-11-14 00:00:00.000000000 Z
+date: 2019-06-14 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: makara
@@ -16,14 +16,14 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '0.3'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '0.3'
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement
@@ -70,16 +70,16 @@ dependencies:
   name: pg
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "<"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '1'
+        version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "<"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '1'
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: mysql2
   requirement: !ruby/object:Gem::Requirement
@@ -109,20 +109,14 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
 description: 
-email:
-- andrew@chartkick.com
+email: andrew@chartkick.com
 executables: []
 extensions: []
 extra_rdoc_files: []
 files:
-- ".gitignore"
-- ".travis.yml"
 - CHANGELOG.md
-- Gemfile
 - LICENSE.txt
 - README.md
-- Rakefile
-- distribute_reads.gemspec
 - lib/distribute_reads.rb
 - lib/distribute_reads/appropriate_pool.rb
 - lib/distribute_reads/cache_store.rb
@@ -141,15 +135,14 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: '0'
+      version: '2.4'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project: 
-rubygems_version: 2.7.6
+rubygems_version: 3.0.3
 signing_key: 
 specification_version: 4
 summary: Scale database reads with replicas in Rails

data/.gitignore

@@ -1,9 +0,0 @@
-/.bundle/
-/.yardoc
-/_yardoc/
-/coverage/
-/doc/
-/pkg/
-/spec/reports/
-/tmp/
-*.lock

data/.travis.yml

@@ -1,14 +0,0 @@
-sudo: false
-language: ruby
-rvm: 2.5.1
-script: bundle exec rake test
-before_script:
-  - psql -c 'create database distribute_reads_test_primary;' -U postgres
-  - psql -c 'create database distribute_reads_test_replica;' -U postgres
-notifications:
-  email:
-    on_success: never
-    on_failure: change
-gemfile:
-  - Gemfile
-  - test/gemfiles/makara3.gemfile

data/Gemfile

@@ -1,4 +0,0 @@
-source "https://rubygems.org"
-
-# Specify your gem's dependencies in distribute_reads.gemspec
-gemspec

data/Rakefile

@@ -1,11 +0,0 @@
-require "bundler/gem_tasks"
-require "rake/testtask"
-
-Rake::TestTask.new(:test) do |t|
-  t.libs << "test"
-  t.libs << "lib"
-  t.test_files = FileList["test/**/*_test.rb"]
-  t.warning = false
-end
-
-task default: :test

data/distribute_reads.gemspec

@@ -1,31 +0,0 @@
-# coding: utf-8
-lib = File.expand_path("../lib", __FILE__)
-$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
-require "distribute_reads/version"
-
-Gem::Specification.new do |spec|
-  spec.name          = "distribute_reads"
-  spec.version       = DistributeReads::VERSION
-  spec.authors       = ["Andrew Kane"]
-  spec.email         = ["andrew@chartkick.com"]
-
-  spec.summary       = "Scale database reads with replicas in Rails"
-  spec.homepage      = "https://github.com/ankane/distribute_reads"
-  spec.license       = "MIT"
-
-  spec.files = `git ls-files -z`.split("\x0").reject do |f|
-    f.match(%r{^(test|spec|features)/})
-  end
-  spec.bindir        = "exe"
-  spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
-  spec.require_paths = ["lib"]
-
-  spec.add_dependency "makara"
-
-  spec.add_development_dependency "bundler"
-  spec.add_development_dependency "rake"
-  spec.add_development_dependency "minitest"
-  spec.add_development_dependency "pg", "< 1"
-  spec.add_development_dependency "mysql2"
-  spec.add_development_dependency "activejob"
-end