distribute_reads 0.3.0 → 0.3.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 4fb973f4af2e8828fc62a2c921605d595ee89bf521ba9f2b65f4c4f695225a76
-  data.tar.gz: 26ac05c9a814401ed03e56dfd7ceafa637e3ac0dcdb8aba79e12f9d664b5bbac
+  metadata.gz: 0e52cc02b09f1027e932869c4acacff1d979bbe1c4deb34c952e81bd14c97fc9
+  data.tar.gz: 783d432ebe59418e717cd52e2b09ae54174a21000f4cdef96be6e0ebf6651c22
 SHA512:
-  metadata.gz: f40892ea166bb86ea89809d7b0e1cb74616ba55251f3208272319dab0d71322eb2c4efca3aaa2c134a9e0ae6adc1c20ea2cae16451d3c2fd6205e4f344cd27a8
-  data.tar.gz: 415fa474c9e3185f9d39b5410747f3d0a9feaf775e9a7e92d1595bc28d94e8ce69caa06d1c36eb646a98c94f963bc16ae27e54780260172689792066cf06ada2
+  metadata.gz: 728d62fd84f564e7e35981ae7bd37177c02ab69304b0a9c36e60ec2045af27a43fb284db4f12f5ca9094cbe9fd8108c3f4b6bed0421f4613bee9fb4a50ca8169
+  data.tar.gz: 3698ec3ba5c50277a638f013293be1c28a0404baf477a0473e135bf1512641ac9afc6f89060a031f9b256bce162e3585b51640a8672e7c8b8b7ddbb8aa2059ee

data/CHANGELOG.md

@@ -1,29 +1,46 @@
-## 0.3.0
+## 0.3.4 (2021-08-22)
+
+- Added support for Aurora Postgres replication lag
+
+## 0.3.3 (2020-05-06)
+
+- Fixed deprecation warning with MySQL
+
+## 0.3.2 (2020-01-02)
+
+- Added `eager_load` option
+- Removed warning when relation is loaded
+
+## 0.3.1 (2019-10-28)
+
+- Added source location to logging
+
+## 0.3.0 (2019-06-14)
 
 - Use logger instead of stderr
 - Handle `NULL` replication lag for MySQL
 - Fixed replication lag check running on primary when replicas blacklisted
 
-## 0.2.4
+## 0.2.4 (2018-11-14)
 
 - Added support for Aurora MySQL replication lag
 - Added more logging
 
-## 0.2.3
+## 0.2.3 (2018-05-24)
 
 - Added support for Makara 0.4
 
-## 0.2.2
+## 0.2.2 (2018-03-29)
 
 - Added support for MySQL replication lag
 - Added `replica` option
 
-## 0.2.1
+## 0.2.1 (2017-12-14)
 
 - Fixed lag check for Postgres 10
 - Added `replication_lag` method
 
-## 0.2.0
+## 0.2.0 (2017-10-02)
 
 Breaking
 
@@ -40,16 +57,16 @@ Other
 - Added default options
 - Improved lag query
 
-## 0.1.2
+## 0.1.2 (2017-09-20)
 
 - Raise `ArgumentError` when missing block
 - Improved lag query
 - Warn if returning `ActiveRecord::Relation`
 
-## 0.1.1
+## 0.1.1 (2017-05-14)
 
 - Added method for jobs
 
-## 0.1.0
+## 0.1.0 (2017-03-26)
 
 - First release

data/LICENSE.txt

@@ -1,4 +1,4 @@
-Copyright (c) 2017-2019 Andrew Kane
+Copyright (c) 2017-2021 Andrew Kane
 
 MIT License
 

data/README.md

@@ -4,7 +4,7 @@ Scale database reads to replicas in Rails
 
 :tangerine: Battle-tested at [Instacart](https://www.instacart.com/opensource)
 
-[![Build Status](https://travis-ci.org/ankane/distribute_reads.svg?branch=master)](https://travis-ci.org/ankane/distribute_reads)
+[![Build Status](https://github.com/ankane/distribute_reads/workflows/build/badge.svg?branch=master)](https://github.com/ankane/distribute_reads/actions)
 
 ## Installation
 
@@ -80,12 +80,18 @@ ActiveRecord uses [lazy evaluation](https://www.theodinproject.com/courses/ruby-
 users = distribute_reads { User.where(orders_count: 1) } # not executed yet
 ```
 
-Call `to_a` inside the block ensure the query runs on a replica.
+Call `to_a` or `load` inside the block to ensure the query runs on a replica.
 
 ```ruby
 users = distribute_reads { User.where(orders_count: 1).to_a }
 ```
 
+You can automatically load relations returned from `distribute_reads` blocks by creating an initializer with:
+
+```ruby
+DistributeReads.eager_load = true
+```
+
 ## Options
 
 ### Replica Lag
@@ -181,7 +187,7 @@ end
 
 ## Rails 6
 
-Rails 6 has [native support for replicas](https://edgeguides.rubyonrails.org/active_record_multiple_databases.html) :tada:
+Rails 6 has [native support for replicas](https://guides.rubyonrails.org/active_record_multiple_databases.html) :tada:
 
 ```ruby
 ActiveRecord::Base.connected_to(role: :reading) do
@@ -208,13 +214,13 @@ Everyone is encouraged to help improve this project. Here are a few ways you can
 - Write, clarify, or fix documentation
 - Suggest or add new features
 
-To test, run:
+To get started with development and testing:
 
 ```sh
 git clone https://github.com/ankane/distribute_reads.git
 cd distribute_reads
 createdb distribute_reads_test_primary
 createdb distribute_reads_test_replica
-bundle
-bundle exec rake
+bundle install
+bundle exec rake test
 ```

data/lib/distribute_reads.rb

@@ -14,8 +14,7 @@ module DistributeReads
   class NoReplicasAvailable < Error; end
 
   class << self
-    attr_accessor :by_default
-    attr_accessor :default_options
+    attr_accessor :by_default, :default_options, :eager_load
     attr_writer :logger
   end
   self.by_default = false
@@ -23,6 +22,7 @@ module DistributeReads
     failover: true,
     lag_failover: false
   }
+  self.eager_load = false
 
   def self.logger
     unless defined?(@logger)
@@ -43,37 +43,57 @@ module DistributeReads
       case connection.adapter_name
       when "PostgreSQL", "PostGIS"
         # cache the version number
-        @server_version_num ||= {}
+        @aurora_postgres ||= {}
         cache_key = connection.pool.object_id
-        @server_version_num[cache_key] ||= connection.execute("SHOW server_version_num").first["server_version_num"].to_i
 
-        lag_condition =
-          if @server_version_num[cache_key] >= 100000
-            "pg_last_wal_receive_lsn() = pg_last_wal_replay_lsn()"
-          else
-            "pg_last_xlog_receive_location() = pg_last_xlog_replay_location()"
+        unless @aurora_postgres.key?(cache_key)
+          @aurora_postgres[cache_key] = connection.select_all("SELECT 1 FROM pg_stat_activity WHERE backend_type = 'aurora runtime'").any?
+        end
+
+        @server_version_num ||= {}
+        @server_version_num[cache_key] ||= connection.select_all("SHOW server_version_num").first["server_version_num"].to_i
+
+        if @aurora_postgres[cache_key]
+          # no way to get session_id at the moment
+          # also, pg_is_in_recovery() is always false
+          # and pg_settings are the same for writer and readers
+          # this means we can't tell:
+          # 1. if this is the primary or replica
+          # 2. if replica, which one
+          status = connection.select_all("SELECT MAX(replica_lag_in_msec) AS replica_lag_in_msec, COUNT(*) AS replica_count FROM aurora_replica_status() WHERE session_id != 'MASTER_SESSION_ID'").first
+          if status && status["replica_count"].to_i > 1
+            log "Multiple readers available, taking max lag of all of them"
           end
+          status ? status["replica_lag_in_msec"].to_f / 1000.0 : 0.0
+        else
+          lag_condition =
+            if @server_version_num[cache_key] >= 100000
+              "pg_last_wal_receive_lsn() = pg_last_wal_replay_lsn()"
+            else
+              "pg_last_xlog_receive_location() = pg_last_xlog_replay_location()"
+            end
 
-        connection.execute(
-          "SELECT CASE
-            WHEN NOT pg_is_in_recovery() OR #{lag_condition} THEN 0
-            ELSE EXTRACT (EPOCH FROM NOW() - pg_last_xact_replay_timestamp())
-          END AS lag".squish
-        ).first["lag"].to_f
+          connection.select_all(
+            "SELECT CASE
+              WHEN NOT pg_is_in_recovery() OR #{lag_condition} THEN 0
+              ELSE EXTRACT (EPOCH FROM NOW() - pg_last_xact_replay_timestamp())
+            END AS lag".squish
+          ).first["lag"].to_f
+        end
       when "MySQL", "Mysql2", "Mysql2Spatial", "Mysql2Rgeo"
         @aurora_mysql ||= {}
         cache_key = connection.pool.object_id
 
         unless @aurora_mysql.key?(cache_key)
           # makara doesn't send SHOW queries to replica by default
-          @aurora_mysql[cache_key] = connection.exec_query("SHOW VARIABLES LIKE 'aurora_version'").to_hash.any?
+          @aurora_mysql[cache_key] = connection.select_all("SHOW VARIABLES LIKE 'aurora_version'").any?
         end
 
         if @aurora_mysql[cache_key]
-          status = connection.exec_query("SELECT Replica_lag_in_msec FROM mysql.ro_replica_status WHERE Server_id = @@aurora_server_id").to_hash.first
+          status = connection.select_all("SELECT Replica_lag_in_msec FROM mysql.ro_replica_status WHERE Server_id = @@aurora_server_id").first
           status ? status["Replica_lag_in_msec"].to_f / 1000.0 : 0.0
         else
-          status = connection.exec_query("SHOW SLAVE STATUS").to_hash.first
+          status = connection.select_all("SHOW SLAVE STATUS").first
           if status
             if status["Seconds_Behind_Master"].nil?
               # replication stopped
@@ -97,7 +117,24 @@ module DistributeReads
   end
 
   def self.log(message)
-    logger.info("[distribute_reads] #{message}") if logger
+    if logger
+      logger.info { "[distribute_reads] #{message}" }
+
+      # show location like Active Record
+      source = backtrace_cleaner.clean(caller.lazy).first
+      logger.info { "  ↳ #{source}" } if source
+    end
+  end
+
+  # private
+  def self.backtrace_cleaner
+    @backtrace_cleaner ||= begin
+      bc = ActiveSupport::BacktraceCleaner.new
+      bc.add_silencer { |line| line.include?("lib/distribute_reads") }
+      bc.add_silencer { |line| line.include?("lib/makara") }
+      bc.add_silencer { |line| line.include?("lib/active_record") }
+      bc
+    end
   end
 
   # private
@@ -132,6 +169,8 @@ end
 
 Makara::Proxy.prepend DistributeReads::AppropriatePool
 Object.include DistributeReads::GlobalMethods
+# TODO uncomment in 0.4.0
+# Object.send :private, :distribute_reads
 
 ActiveSupport.on_load(:active_job) do
   require "distribute_reads/job_methods"

data/lib/distribute_reads/global_methods.rb

@@ -23,7 +23,7 @@ module DistributeReads
             current_lag =
               begin
                 DistributeReads.replication_lag(connection: base_model.connection)
-              rescue DistributeReads::NoReplicasAvailable => e
+              rescue DistributeReads::NoReplicasAvailable
                 # TODO rescue more exceptions?
                 false
               end
@@ -54,7 +54,13 @@ module DistributeReads
         end
 
         value = yield
-        DistributeReads.log "Call `to_a` inside block to execute query on replica" if value.is_a?(ActiveRecord::Relation) && !previous_value
+        if value.is_a?(ActiveRecord::Relation) && !previous_value && !value.loaded?
+          if DistributeReads.eager_load
+            value = value.load
+          else
+            DistributeReads.log "Call `to_a` inside block to execute query on replica"
+          end
+        end
         value
       ensure
         Thread.current[:distribute_reads] = previous_value

data/lib/distribute_reads/version.rb

@@ -1,3 +1,3 @@
 module DistributeReads
-  VERSION = "0.3.0"
+  VERSION = "0.3.4"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: distribute_reads
 version: !ruby/object:Gem::Version
-  version: 0.3.0
+  version: 0.3.4
 platform: ruby
 authors:
 - Andrew Kane
-autorequire: 
+autorequire:
 bindir: bin
 cert_chain: []
-date: 2019-06-14 00:00:00.000000000 Z
+date: 2021-08-22 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: makara
@@ -24,92 +24,8 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0.3'
-- !ruby/object:Gem::Dependency
-  name: bundler
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-- !ruby/object:Gem::Dependency
-  name: rake
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-- !ruby/object:Gem::Dependency
-  name: minitest
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-- !ruby/object:Gem::Dependency
-  name: pg
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-- !ruby/object:Gem::Dependency
-  name: mysql2
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-- !ruby/object:Gem::Dependency
-  name: activejob
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-description: 
-email: andrew@chartkick.com
+description:
+email: andrew@ankane.org
 executables: []
 extensions: []
 extra_rdoc_files: []
@@ -127,7 +43,7 @@ homepage: https://github.com/ankane/distribute_reads
 licenses:
 - MIT
 metadata: {}
-post_install_message: 
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -142,8 +58,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.0.3
-signing_key: 
+rubygems_version: 3.2.22
+signing_key:
 specification_version: 4
 summary: Scale database reads with replicas in Rails
 test_files: []