active_record_host_pool 1.0.2 → 1.1.1

data/test/database.yml

@@ -1,76 +1,107 @@
 <% mysql = URI(ENV['MYSQL_URL'] || 'mysql://root@127.0.0.1:3306') %>
-test_host_1_db_1:
+# ARHP creates separate connection pools based on the pool key.
+# The pool key is defined as:
+#   host / port / socket / username / replica
+#
+# Therefore two databases with identical host, port, socket, username, and replica status will share a connection pool.
+# If any part (host, port, etc.) of the pool key differ, two databases will _not_ share a connection pool.
+#
+# Below, "test_pool_1..." and "test_pool_2..." have identical host, username, socket, and replica status but the port information differs.
+# Here the yml configurations are reformatted as a table to give a visual example:
+#
+# |----------+----------------+----------------|
+# |          |  test_pool_1   |  test_pool_2   |
+# |----------+----------------+----------------+
+# | host     | 127.0.0.1      | 127.0.0.1      |
+# | port     |                | 3306           |
+# | socket   |                |                |
+# | username | root           | root           |
+# | replica  | false          | false          |
+# |----------+----------------+----------------|
+#
+# Note: The configuration items must be explicitly defined or will be blank in the pool key.
+#       Configurations with matching _implicit_ items but differing _explicit_ items will create separate pools.
+#       e.g. "test_pool_1" will default to port 3306 but because it is not explicitly defined it will not share a pool with test_pool_2
+#
+#       ARHP will therefore create the following pool keys:
+#       test_pool_1 => 127.0.0.1///root/false
+#       test_pool_2 => 127.0.0.1/3306//root/false
+
+test_pool_1_db_a:
   adapter: mysql2
   encoding: utf8
-  database: arhp_test_1
+  database: arhp_test_db_a
   username: <%= mysql.user %>
   password: <%= mysql.password %>
   host: <%= mysql.host %>
   reconnect: true
 
 # Mimic configurations as read by active_record_shards/ar_flexmaster
-test_host_1_db_1_slave:
+test_pool_1_db_a_replica:
   adapter: mysql2
   encoding: utf8
-  database: arhp_test_1_slave
+  database: arhp_test_db_a_replica
   username: <%= mysql.user %>
   password: <%= mysql.password %>
   host: <%= mysql.host %>
   reconnect: true
   slave: true
 
-test_host_1_db_2:
+test_pool_1_db_b:
   adapter: mysql2
   encoding: utf8
-  database: arhp_test_2
+  database: arhp_test_db_b
   username: <%= mysql.user %>
   password: <%= mysql.password %>
   host: <%= mysql.host %>
   reconnect: true
 
-test_host_2_db_3:
+test_pool_1_db_not_there:
   adapter: mysql2
   encoding: utf8
-  database: arhp_test_3
+  database: arhp_test_db_not_there
   username: <%= mysql.user %>
   password: <%= mysql.password %>
   host: <%= mysql.host %>
-  port: <%= mysql.port %>
   reconnect: true
 
-test_host_2_db_4:
+test_pool_2_db_d:
   adapter: mysql2
   encoding: utf8
-  database: arhp_test_4
+  database: arhp_test_db_d
   username: <%= mysql.user %>
   password: <%= mysql.password %>
   host: <%= mysql.host %>
   port: <%= mysql.port %>
   reconnect: true
 
-test_host_2_db_5:
+test_pool_2_db_e:
   adapter: mysql2
   encoding: utf8
-  database: arhp_test_4
-  username: john-doe
-  password:
+  database: arhp_test_db_e
+  username: <%= mysql.user %>
+  password: <%= mysql.password %>
   host: <%= mysql.host %>
   port: <%= mysql.port %>
   reconnect: true
 
-test_host_1_db_not_there:
+test_pool_3_db_e:
   adapter: mysql2
   encoding: utf8
-  database: arhp_test_no_create
-  username: <%= mysql.user %>
-  password: <%= mysql.password %>
+  database: arhp_test_db_e
+  username: john-doe
+  password:
   host: <%= mysql.host %>
+  port: <%= mysql.port %>
   reconnect: true
 
-test_host_1_db_shard:
+# test_pool_1_db_c needs to be the last database defined in the file
+# otherwise the test_models_with_matching_hosts_and_non_matching_databases_issue_exists_without_arhp_patch
+# test fails
+test_pool_1_db_c:
   adapter: mysql2
   encoding: utf8
-  database: arhp_test_1_shard
+  database: arhp_test_db_c
   username: <%= mysql.user %>
   password: <%= mysql.password %>
   host: <%= mysql.host %>

data/test/helper.rb

@@ -5,15 +5,41 @@ require 'minitest/autorun'
 
 require 'active_record_host_pool'
 require 'logger'
-require 'mocha/setup'
+require 'mocha/minitest'
 require 'phenix'
 
-RAILS_ENV = 'test'
+ENV['RAILS_ENV'] = 'test'
+ENV['LEGACY_CONNECTION_HANDLING'] = 'true' if ENV['LEGACY_CONNECTION_HANDLING'].nil?
 
-Minitest::Test = MiniTest::Unit::TestCase unless defined?(::Minitest::Test)
+if ActiveRecord.version >= Gem::Version.new('6.1')
+  ActiveRecord::Base.legacy_connection_handling = (ENV['LEGACY_CONNECTION_HANDLING'] == 'true')
+end
+
+RAILS_6_1_WITH_NON_LEGACY_CONNECTION_HANDLING =
+  ActiveRecord.version >= Gem::Version.new('6.1') && !ActiveRecord::Base.legacy_connection_handling
 
 ActiveRecord::Base.logger = Logger.new(File.dirname(__FILE__) + '/test.log')
 
+# BEGIN preventing_writes? patch
+## Rails 6.1 by default does not allow writing to replica databases which prevents
+## us from properly setting up the test databases. This patch is used in test/schema.rb
+## to allow us to write to the replicas but only during migrations
+module ActiveRecordHostPool
+  cattr_accessor :allowing_writes
+  module PreventWritesPatch
+    def preventing_writes?
+      return false if ActiveRecordHostPool.allowing_writes && replica?
+
+      super
+    end
+  end
+end
+
+if ActiveRecord.version >= Gem::Version.new('6.1')
+  ActiveRecord::ConnectionAdapters::AbstractAdapter.prepend ActiveRecordHostPool::PreventWritesPatch
+end
+# END preventing_writes? patch
+
 Phenix.configure do |config|
   config.skip_database = ->(name, conf) { name =~ /not_there/ || conf['username'] == 'john-doe' }
 end
@@ -22,46 +48,121 @@ module ARHPTestSetup
   private
 
   def arhp_create_models
-    return if ARHPTestSetup.const_defined?('Test1')
-
-    eval <<-RUBY
-      # The placement of the Test1Shard class is important so that its
-      # connection will not be the most recent connection established
-      # for test_host_1.
-      class Test1Shard < ::ActiveRecord::Base
-        establish_connection(:test_host_1_db_shard)
-      end
-
-      class Test1 < ActiveRecord::Base
-        self.table_name = "tests"
-        establish_connection(:test_host_1_db_1)
-      end
-
-      class Test1Slave < ActiveRecord::Base
-        self.table_name = "tests"
-        establish_connection(:test_host_1_db_1_slave)
-      end
-
-      class Test2 < ActiveRecord::Base
-        self.table_name =  "tests"
-        establish_connection(:test_host_1_db_2)
-      end
-
-      class Test3 < ActiveRecord::Base
-        self.table_name = "tests"
-        establish_connection(:test_host_2_db_3)
-      end
-
-      class Test4 < ActiveRecord::Base
-        self.table_name = "tests"
-        establish_connection(:test_host_2_db_4)
-      end
-
-      class Test5 < ActiveRecord::Base
-        self.table_name = "tests"
-        establish_connection(:test_host_2_db_5)
-      end
-    RUBY
+    return if ARHPTestSetup.const_defined?('Pool1DbA')
+
+    if RAILS_6_1_WITH_NON_LEGACY_CONNECTION_HANDLING
+      eval <<-RUBY
+        class AbstractPool1DbC < ActiveRecord::Base
+          self.abstract_class = true
+          connects_to database: { writing: :test_pool_1_db_c }
+        end
+
+        # The placement of the Pool1DbC class is important so that its
+        # connection will not be the most recent connection established
+        # for test_pool_1.
+        class Pool1DbC < AbstractPool1DbC
+        end
+
+        class AbstractPool1DbA < ActiveRecord::Base
+          self.abstract_class = true
+          connects_to database: { writing: :test_pool_1_db_a, reading: :test_pool_1_db_a_replica }
+        end
+
+        class Pool1DbA < AbstractPool1DbA
+          self.table_name = "tests"
+        end
+
+        class AbstractPool1DbB < ActiveRecord::Base
+          self.abstract_class = true
+          connects_to database: { writing: :test_pool_1_db_b }
+        end
+
+        class Pool1DbB < AbstractPool1DbB
+          self.table_name = "tests"
+        end
+
+        class AbstractPool2DbD < ActiveRecord::Base
+          self.abstract_class = true
+          connects_to database: { writing: :test_pool_2_db_d }
+        end
+
+        class Pool2DbD < AbstractPool2DbD
+          self.table_name = "tests"
+        end
+
+        class AbstractPool2DbE < ActiveRecord::Base
+          self.abstract_class = true
+          connects_to database: { writing: :test_pool_2_db_e }
+        end
+
+        class Pool2DbE < AbstractPool2DbE
+          self.table_name = "tests"
+        end
+
+        class AbstractPool3DbE < ActiveRecord::Base
+          self.abstract_class = true
+          connects_to database: { writing: :test_pool_3_db_e }
+        end
+
+        class Pool3DbE < AbstractPool3DbE
+          self.table_name = "tests"
+        end
+
+        # Test ARHP with Rails 6.1+ horizontal sharding functionality
+        class AbstractShardedModel < ActiveRecord::Base
+          self.abstract_class = true
+          connects_to shards: {
+                        default: { writing: :test_pool_1_db_shard_a },
+                        shard_b: { writing: :test_pool_1_db_shard_b, reading: :test_pool_1_db_shard_b_replica },
+                        shard_c: { writing: :test_pool_1_db_shard_c, reading: :test_pool_1_db_shard_c_replica },
+                        shard_d: { writing: :test_pool_2_db_shard_d, reading: :test_pool_2_db_shard_d_replica }
+                      }
+        end
+
+        class ShardedModel < AbstractShardedModel
+          self.table_name = "tests"
+        end
+      RUBY
+    else
+      eval <<-RUBY
+        # The placement of the Pool1DbC class is important so that its
+        # connection will not be the most recent connection established
+        # for test_pool_1.
+        class Pool1DbC < ActiveRecord::Base
+          establish_connection(:test_pool_1_db_c)
+        end
+
+        class Pool1DbA < ActiveRecord::Base
+          self.table_name = "tests"
+          establish_connection(:test_pool_1_db_a)
+        end
+
+        class Pool1DbAReplica < ActiveRecord::Base
+          self.table_name = "tests"
+          establish_connection(:test_pool_1_db_a_replica)
+        end
+
+        class Pool1DbB < ActiveRecord::Base
+          self.table_name =  "tests"
+          establish_connection(:test_pool_1_db_b)
+        end
+
+        class Pool2DbD < ActiveRecord::Base
+          self.table_name = "tests"
+          establish_connection(:test_pool_2_db_d)
+        end
+
+        class Pool2DbE < ActiveRecord::Base
+          self.table_name = "tests"
+          establish_connection(:test_pool_2_db_e)
+        end
+
+        class Pool3DbE < ActiveRecord::Base
+          self.table_name = "tests"
+          establish_connection(:test_pool_3_db_e)
+        end
+      RUBY
+    end
   end
 
   def current_database(klass)
@@ -78,4 +179,17 @@ module ARHPTestSetup
   ensure
     mod.define_method(method_name, method_body)
   end
+
+  def simulate_rails_app_active_record_railties
+    if ActiveRecord.version >= Gem::Version.new('6.0') && !RAILS_6_1_WITH_NON_LEGACY_CONNECTION_HANDLING
+      # Necessary for testing ActiveRecord 6.0 which uses the connection
+      # handlers when clearing query caches across all handlers when
+      # an operation that dirties the cache is involved (e.g. create/insert,
+      # update, delete/destroy, truncate, etc.)
+      # In Rails 6.1 this is only present when legacy_connection_handling=true
+      ActiveRecord::Base.connection_handlers = {
+        ActiveRecord::Base.writing_role => ActiveRecord::Base.default_connection_handler
+      }
+    end
+  end
 end

data/test/schema.rb

@@ -1,16 +1,23 @@
 # frozen_string_literal: true
 
 require_relative 'helper'
-ActiveRecord::Schema.define(version: 1) do
-  create_table 'tests', force: true do |t|
-    t.string   'val'
-  end
 
-  # Add a table only the shard database will have. Conditional
-  # exists since Phenix loads the schema file for every database.
-  if ActiveRecord::Base.connection.current_database == 'arhp_test_1_shard'
-    create_table 'test1_shards' do |t|
-      t.string 'name'
+begin
+  ActiveRecordHostPool.allowing_writes = true
+
+  ActiveRecord::Schema.define(version: 1) do
+    create_table 'tests', force: true do |t|
+      t.string   'val'
+    end
+
+    # Add a table only the shard database will have. Conditional
+    # exists since Phenix loads the schema file for every database.
+    if ActiveRecord::Base.connection.current_database == 'arhp_test_db_c'
+      create_table 'pool1_db_cs' do |t|
+        t.string 'name'
+      end
     end
   end
+ensure
+  ActiveRecordHostPool.allowing_writes = false
 end

data/test/test_arhp.rb

@@ -5,11 +5,17 @@ require_relative 'helper'
 class ActiveRecordHostPoolTest < Minitest::Test
   include ARHPTestSetup
   def setup
-    Phenix.rise!
+    if RAILS_6_1_WITH_NON_LEGACY_CONNECTION_HANDLING
+      Phenix.rise! config_path: 'test/three_tier_database.yml'
+    else
+      Phenix.rise!
+    end
     arhp_create_models
   end
 
   def teardown
+    ActiveRecord::Base.connection.disconnect!
+    ActiveRecordHostPool::PoolProxy.class_variable_set(:@@_connection_pools, {})
     Phenix.burn!
   end
 
@@ -30,110 +36,80 @@ class ActiveRecordHostPoolTest < Minitest::Test
     ActiveRecord::Base.connection_handler.clear_all_connections!
   end
 
-  def test_models_with_matching_hosts_should_share_a_connection
-    assert_equal(Test1.connection.raw_connection, Test2.connection.raw_connection)
-    assert_equal(Test3.connection.raw_connection, Test4.connection.raw_connection)
-  end
-
-  def test_models_without_matching_hosts_should_not_share_a_connection
-    refute_equal(Test1.connection.raw_connection, Test4.connection.raw_connection)
+  def test_models_with_matching_hosts_ports_sockets_usernames_and_replica_status_should_share_a_connection
+    assert_equal(Pool1DbA.connection.raw_connection, Pool1DbB.connection.raw_connection)
+    assert_equal(Pool2DbD.connection.raw_connection, Pool2DbE.connection.raw_connection)
   end
 
-  def test_models_without_matching_usernames_should_not_share_a_connection
-    refute_equal(Test4.connection.raw_connection, Test5.connection.raw_connection)
+  def test_models_with_different_ports_should_not_share_a_connection
+    refute_equal(Pool1DbA.connection.raw_connection, Pool2DbD.connection.raw_connection)
   end
 
-  def test_models_without_match_slave_status_should_not_share_a_connection
-    refute_equal(Test1.connection.raw_connection, Test1Slave.connection.raw_connection)
+  def test_models_with_different_usernames_should_not_share_a_connection
+    refute_equal(Pool2DbE.connection.raw_connection, Pool3DbE.connection.raw_connection)
   end
 
   def test_should_select_on_correct_database
-    assert_action_uses_correct_database(:select_all, 'select 1')
-  end
+    Pool1DbA.connection.send(:select_all, 'select 1')
+    assert_equal 'arhp_test_db_a', current_database(Pool1DbA)
 
-  def test_should_insert_on_correct_database
-    assert_action_uses_correct_database(:insert, "insert into tests values(NULL, 'foo')")
-  end
+    Pool2DbD.connection.send(:select_all, 'select 1')
+    assert_equal 'arhp_test_db_d', current_database(Pool2DbD)
 
-  def test_models_with_matching_hosts_and_non_matching_databases_should_share_a_connection
-    simulate_rails_app_active_record_railties
-    assert_equal(Test1.connection.raw_connection, Test1Shard.connection.raw_connection)
+    Pool3DbE.connection.send(:select_all, 'select 1')
+    assert_equal 'arhp_test_db_e', current_database(Pool3DbE)
   end
 
-  if ActiveRecord.version >= Gem::Version.new('6.0')
-    def test_models_with_matching_hosts_and_non_matching_databases_issue_exists_without_arhp_patch
-      simulate_rails_app_active_record_railties
-
-      # Remove patch that fixes an issue in Rails 6+ to ensure it still
-      # exists. If this begins to fail then it may mean that Rails has fixed
-      # the issue so that it no longer occurs.
-      without_module_patch(ActiveRecordHostPool::ClearQueryCachePatch, :clear_query_caches_for_current_thread) do
-        exception = assert_raises(ActiveRecord::StatementInvalid) do
-          ActiveRecord::Base.cache { Test1Shard.create! }
-        end
-
-        assert_equal("Mysql2::Error: Table 'arhp_test_2.test1_shards' doesn't exist", exception.message)
-      end
-    end
+  def test_should_insert_on_correct_database
+    Pool1DbA.connection.send(:insert, "insert into tests values(NULL, 'foo')")
+    assert_equal 'arhp_test_db_a', current_database(Pool1DbA)
 
-    def test_models_with_matching_hosts_and_non_matching_databases_do_not_mix_up_underlying_database
-      simulate_rails_app_active_record_railties
+    Pool2DbD.connection.send(:insert, "insert into tests values(NULL, 'foo')")
+    assert_equal 'arhp_test_db_d', current_database(Pool2DbD)
 
-      # ActiveRecord 6.0 introduced a change that surfaced a problematic code
-      # path in active_record_host_pool when clearing caches across connection
-      # handlers which can cause the database to change.
-      # See ActiveRecordHostPool::ClearQueryCachePatch
-      ActiveRecord::Base.cache { Test1Shard.create! }
-    end
+    Pool3DbE.connection.send(:insert, "insert into tests values(NULL, 'foo')")
+    assert_equal 'arhp_test_db_e', current_database(Pool3DbE)
   end
 
   def test_connection_returns_a_proxy
-    assert_kind_of ActiveRecordHostPool::ConnectionProxy, Test1.connection
+    assert_kind_of ActiveRecordHostPool::ConnectionProxy, Pool1DbA.connection
   end
 
   def test_connection_proxy_handles_private_methods
     # Relies on connection.class returning the real class
-    Test1.connection.class.class_eval do
+    Pool1DbA.connection.class.class_eval do
       private
 
       def test_private_method
         true
       end
     end
-    assert Test1.connection.respond_to?(:test_private_method, true)
-    refute Test1.connection.respond_to?(:test_private_method)
-    assert_includes(Test1.connection.private_methods, :test_private_method)
-    assert_equal true, Test1.connection.send(:test_private_method)
-  end
-
-  def test_should_not_share_a_query_cache
-    Test1.create(val: 'foo')
-    Test2.create(val: 'foobar')
-    Test1.connection.cache do
-      refute_equal Test1.first.val, Test2.first.val
-    end
+    assert Pool1DbA.connection.respond_to?(:test_private_method, true)
+    refute Pool1DbA.connection.respond_to?(:test_private_method)
+    assert_includes(Pool1DbA.connection.private_methods, :test_private_method)
+    assert_equal true, Pool1DbA.connection.send(:test_private_method)
   end
 
   def test_object_creation
-    Test1.create(val: 'foo')
-    assert_equal('arhp_test_1', current_database(Test1))
+    Pool1DbA.create(val: 'foo')
+    assert_equal('arhp_test_db_a', current_database(Pool1DbA))
 
-    Test3.create(val: 'bar')
-    assert_equal('arhp_test_1', current_database(Test1))
-    assert_equal('arhp_test_3', current_database(Test3))
+    Pool2DbD.create(val: 'bar')
+    assert_equal('arhp_test_db_a', current_database(Pool1DbA))
+    assert_equal('arhp_test_db_d', current_database(Pool2DbD))
 
-    Test2.create!(val: 'bar_distinct')
-    assert_equal('arhp_test_2', current_database(Test2))
-    assert Test2.find_by_val('bar_distinct')
-    refute Test1.find_by_val('bar_distinct')
+    Pool1DbB.create!(val: 'bar_distinct')
+    assert_equal('arhp_test_db_b', current_database(Pool1DbB))
+    assert Pool1DbB.find_by_val('bar_distinct')
+    refute Pool1DbA.find_by_val('bar_distinct')
   end
 
   def test_disconnect
-    Test1.create(val: 'foo')
-    unproxied = Test1.connection.unproxied
-    Test1.connection_handler.clear_all_connections!
-    Test1.create(val: 'foo')
-    assert(unproxied != Test1.connection.unproxied)
+    Pool1DbA.create(val: 'foo')
+    unproxied = Pool1DbA.connection.unproxied
+    Pool1DbA.connection_handler.clear_all_connections!
+    Pool1DbA.create(val: 'foo')
+    assert(unproxied != Pool1DbA.connection.unproxied)
   end
 
   def test_checkout
@@ -145,7 +121,7 @@ class ActiveRecordHostPoolTest < Minitest::Test
   end
 
   def test_no_switch_when_creating_db
-    conn = Test1.connection
+    conn = Pool1DbA.connection
     conn.expects(:execute_without_switching)
     conn.expects(:_switch_connection).never
     assert conn._host_pool_current_database
@@ -153,7 +129,7 @@ class ActiveRecordHostPoolTest < Minitest::Test
   end
 
   def test_no_switch_when_dropping_db
-    conn = Test1.connection
+    conn = Pool1DbA.connection
     conn.expects(:execute_without_switching)
     conn.expects(:_switch_connection).never
     assert conn._host_pool_current_database
@@ -163,17 +139,17 @@ class ActiveRecordHostPoolTest < Minitest::Test
   def test_underlying_assumption_about_test_db
     debug_me = false
     # ensure connection
-    Test1.first
+    Pool1DbA.first
 
     # which is the "default" DB to connect to?
-    first_db = Test1.connection.unproxied.instance_variable_get(:@_cached_current_database)
+    first_db = Pool1DbA.connection.unproxied.instance_variable_get(:@_cached_current_database)
     puts "\nOk, we started on #{first_db}" if debug_me
 
     switch_to_klass = case first_db
-    when 'arhp_test_2'
-      Test1
-    when 'arhp_test_1'
-      Test2
+    when 'arhp_test_db_b'
+      Pool1DbA
+    when 'arhp_test_db_a'
+      Pool1DbB
     else
       raise "Expected a database name, got #{first_db.inspect}"
     end
@@ -188,7 +164,7 @@ class ActiveRecordHostPoolTest < Minitest::Test
 
     # now, disable our auto-switching and trigger a mysql reconnect
     switch_to_klass.connection.unproxied.stubs(:_switch_connection).returns(true)
-    Test3.connection.execute("KILL #{thread_id}")
+    Pool2DbD.connection.execute("KILL #{thread_id}")
 
     # and finally, did mysql reconnect correctly?
     puts "\nAnd now we end up on #{current_database(switch_to_klass)}" if debug_me
@@ -201,27 +177,4 @@ class ActiveRecordHostPoolTest < Minitest::Test
     pool.expects(:checkin).with(conn)
     pool.release_connection
   end
-
-  private
-
-  def assert_action_uses_correct_database(action, sql)
-    (1..4).each do |i|
-      klass = ARHPTestSetup.const_get("Test#{i}")
-      desired_db = "arhp_test_#{i}"
-      klass.connection.send(action, sql)
-      assert_equal desired_db, current_database(klass)
-    end
-  end
-
-  def simulate_rails_app_active_record_railties
-    if ActiveRecord.version >= Gem::Version.new('6.0')
-      # Necessary for testing ActiveRecord 6.0 which uses the connection
-      # handlers when clearing query caches across all handlers when
-      # an operation that dirties the cache is involved (e.g. create/insert,
-      # update, delete/destroy, truncate, etc.)
-      ActiveRecord::Base.connection_handlers = {
-        ActiveRecord::Base.writing_role => ActiveRecord::Base.default_connection_handler
-      }
-    end
-  end
 end