active_record_shards 4.0.0.beta8 → 5.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: d318eea5c3e064fbc8abc478dbe78c3f173634e3dfb913727c212222c6aadaed
-  data.tar.gz: ee3ccf1174544bbc076ce9470bf67ead36f26e44234368fe6e75a23ba1fbe3f4
+  metadata.gz: aa6b8faea89614322485e584a2e63f9cb7b6ee505c641c68dcf239337776230a
+  data.tar.gz: eb5038c548eecf373a19e811d0354df3c535b6f6f9bfa5cca9140e6d4dbe2994
 SHA512:
-  metadata.gz: cf3e9bfff048f08174f36e2d36811975ff18ed983b018d04e2baa68ee3df8d9da146f9106582583d720c30c98420b38fb85b075562a7746456bafdeefc6110f9
-  data.tar.gz: ca0a7afd7e2fc3e3d6083da8da213b61c5d98c669da3728bf7111e7e054d5d7aa6fb6620b851e8781b8c36db7b746503f171116cb54030f2f715242637ecc5c6
+  metadata.gz: 3b06434d4f484e2d8ea3cceaa98e535712c41b7ac9b2fece389ca25a21e1d0f6249b5c4e85a6206b8718784b9150477b26247de8f667439486dab8d0abf4ef30
+  data.tar.gz: 6453444d52019b1f296df12d96085abaabc40af5bf7595ed2077104efcec8bc7dcda7d43164efb254d09922c67690584e68f6c711d74cefe6de4d16fbe390bab

data/README.md

@@ -1,11 +1,22 @@
-[![Build Status](https://secure.travis-ci.org/zendesk/active_record_shards.png)](http://travis-ci.org/zendesk/active_record_shards)
+[![Build Status](https://github.com/zendesk/active_record_shards/workflows/CI/badge.svg)](https://github.com/zendesk/active_record_shards/actions?query=workflow%3ACI)
 
 # ActiveRecord Shards
 
-ActiveRecord Shards is an extension for ActiveRecord that provides support for sharded database and slaves. Basically it is just a nice way to
+ActiveRecord Shards is an extension for ActiveRecord that provides support for sharded database and replicas. Basically it is just a nice way to
 switch between database connections. We've made the implementation very small, and have tried not to reinvent any wheels already present in ActiveRecord.
 
-ActiveRecord Shards has been used and tested on Rails 5.0 and has in some form or another been used in production on a large Rails app for several years.
+ActiveRecord Shards has been used and tested on Rails 5.x and 6.0, and has in some form or another been used in production on large Rails apps for several years.
+
+Rails 6.1 introduced new connection handling and support for sharding. Apps are encouraged to migrate to the native sharding logic but ActiveRecord Shards supports Rails 6.1 when `legacy_connection_handling` is set to `true`. For more information see [Rails 6.1 installation](#rails-61-installation) and Rails' [multiple databases guide](https://guides.rubyonrails.org/active_record_multiple_databases.html).
+
+- [Installation](#installation)
+- [Configuration](#configuration)
+- [Migrations](#migrations)
+  - [Example](#example)
+    - [Shared Model](#create-a-table-for-the-shared-not-sharded-model)
+    - [Sharded Model](#create-a-table-for-the-sharded-model)
+- [Usage](#usage)
+- [Debugging](#debugging)
 
 ## Installation
 
@@ -13,85 +24,161 @@ ActiveRecord Shards has been used and tested on Rails 5.0 and has in some form o
 
 and make sure to require 'active\_record\_shards' in some way.
 
+### Rails 6.1 installation
+
+Rails 6.1 is **only** supported with `legacy_connection_handling` set to `true`.
+
+Enable the legacy handling in your configuration files e.g. `config/application.rb` by setting:
+
+``` Ruby
+config.active_record.legacy_connection_handling = true
+```
+
+or
+
+``` Ruby
+ActiveRecord::Base.legacy_connection_handling = true
+```
+
 ## Configuration
 
-Add the slave and shard configuration to config/database.yml:
-
-    production:
-      adapter: mysql
-      encoding: utf8
-      database: my_app_main
-      pool: 5
-      host: db1
-      username: root
-      password:
-      slave:
-        host: db1_slave
-      shards:
-        1:
-          host: db_shard1
-          database: my_app_shard
-          slave:
-            host: db_shard1_slave
-        2:
-          host: db_shard2
-          database: my_app_shard
-          slave:
-            host: db_shard2_slave
+Add the replica and shard configuration to config/database.yml:
+
+```yaml
+production:
+  adapter: mysql
+  encoding: utf8
+  database: my_app_main
+  pool: 5
+  host: db1
+  username: root
+  password:
+  replica:
+    host: db1_replica
+  shards:
+    1:
+      host: db_shard1
+      database: my_app_shard
+      replica:
+        host: db_shard1_replica
+    2:
+      host: db_shard2
+      database: my_app_shard
+      replica:
+        host: db_shard2_replica
+```
 
 basically connections inherit configuration from the parent configuration file.
 
-## Usage
+## Migrations
 
-Normally you have some models that live on a shared database, and you might need to query this data in order to know what shard to switch to.
-All the models that live on the sharded database must inherit from ActiveRecordShards::ShardedModel:
+ActiveRecord Shards also patches migrations to support running migrations on a shared (not sharded) or a sharded database.
+Each migration class has to specify a shard spec indicating where to run the migration.
+
+Valid shard specs:
+
+* `:none` - Run this migration on the shared database, not any shards
+* `:all` - Run this migration on all of the shards, not the shared database
+
+#### Example
 
-    class Account < ActiveRecord::Base
-      has_many :projects
+###### Create a table for the shared (not sharded) model
+
+```ruby
+class CreateAccounts < ActiveRecord::Migration
+  shard :none
+
+  def change
+    create_table :accounts do |t|
+      # This is NOT necessary for the gem to work, we just use it in the examples below demonstrating one way to switch shards
+      t.integer :shard_id, null: false
+
+      t.string :name
     end
+  end
+end
+```
+
+###### Create a table for the sharded model
 
-    class Project < ActiveRecordShards::ShardedModel
-      belongs_to :account
+```ruby
+class CreateProjects < ActiveRecord::Migration
+  shard :all
+
+  def change
+    create_table :projects do |t|
+      t.references :account
+      t.string :name
     end
+  end
+end
+```
+
+## Usage
+
+Normally you have some models that live on a shared database, and you might need to query this data in order to know what shard to switch to.
+All the models that live on the shared database must be marked as not\_sharded:
+
+```ruby
+class Account < ActiveRecord::Base
+  not_sharded
+
+  has_many :projects
+end
+
+class Project < ActiveRecord::Base
+  belongs_to :account
+end
+```
 
 So in this setup the accounts live on the shared database, but the projects are sharded. If accounts have a shard\_id column, you could lookup the account
 in a rack middleware and switch to the right shard:
 
-    class AccountMiddleware
-      def initialize(app)
-        @app = app
-      end
-
-      def call(env)
-        account = lookup_account(env)
+```ruby
+class AccountMiddleware
+  def initialize(app)
+    @app = app
+  end
 
-        if account
-          ActiveRecord::Base.on_shard(account.shard_id) do
-            @app.call(env)
-          end
-        else
-          @app.call(env)
-        end
-      end
+  def call(env)
+    account = lookup_account(env)
 
-      def lookup_account(env)
-        ...
+    if account
+      ActiveRecord::Base.on_shard(account.shard_id) do
+        @app.call(env)
       end
+    else
+      @app.call(env)
     end
+  end
 
-You can switch to the slave databases at any point by wrapping your code in an on\_slave block:
+  def lookup_account(env)
+    # ...
+  end
+end
+```
 
-    ActiveRecord::Base.on_slave do
-      Account.find_by_big_expensive_query
-    end
+You can switch to the replica databases at any point by wrapping your code in an on\_replica block:
+
+```ruby
+ActiveRecord::Base.on_replica do
+  Account.find_by_big_expensive_query
+end
+```
+
+This will perform the query on the replica, and mark the returned instances as read-only. There is also a shortcut for this:
+
+```ruby
+Account.on_replica.find_by_big_expensive_query
+```
 
-This will perform the query on the slave, and mark the returned instances as read only. There is also a shortcut for this:
+If you do not want instances returned from replicas to be marked as read-only, this can be disabled globally:
 
-    Account.on_slave.find_by_big_expensive_query
+`ActiveRecordShards.disable_replica_readonly_records = true`
 
 ## Debugging
 
-Show if a query went to master or slave in the logs:
+Show if a query went to primary or replica in the logs:
 
 ```Ruby
 require 'active_record_shards/sql_comments'

data/lib/active_record_shards/association_collection_connection_selection.rb

@@ -1,40 +1,42 @@
 # frozen_string_literal: true
+
 module ActiveRecordShards
   module AssociationCollectionConnectionSelection
-    def on_slave_if(condition)
-      condition ? on_slave : self
+    def on_replica_if(condition)
+      condition ? on_replica : self
     end
 
-    def on_slave_unless(condition)
-      on_slave_if(!condition)
+    def on_replica_unless(condition)
+      on_replica_if(!condition)
     end
 
-    def on_master_if(condition)
-      condition ? on_master : self
+    def on_primary_if(condition)
+      condition ? on_primary : self
     end
 
-    def on_master_unless(condition)
-      on_master_if(!condition)
+    def on_primary_unless(condition)
+      on_primary_if(!condition)
     end
 
-    def on_slave
-      MasterSlaveProxy.new(self, :slave)
+    def on_replica
+      PrimaryReplicaProxy.new(self, :replica)
     end
 
-    def on_master
-      MasterSlaveProxy.new(self, :master)
+    def on_primary
+      PrimaryReplicaProxy.new(self, :primary)
     end
 
-    class MasterSlaveProxy
+    class PrimaryReplicaProxy
       def initialize(association_collection, which)
         @association_collection = association_collection
         @which = which
       end
 
-      def method_missing(method, *args, &block) # rubocop:disable Style/MethodMissing
+      def method_missing(method, *args, &block) # rubocop:disable Style/MethodMissingSuper, Style/MissingRespondToMissing
         reflection = @association_collection.proxy_association.reflection
         reflection.klass.on_cx_switch_block(@which) { @association_collection.send(method, *args, &block) }
       end
+      ruby2_keywords(:method_missing) if respond_to?(:ruby2_keywords, true)
     end
   end
 end

data/lib/active_record_shards/configuration_parser.rb

@@ -1,4 +1,5 @@
 # frozen_string_literal: true
+
 require 'active_support/core_ext'
 
 module ActiveRecordShards
@@ -6,7 +7,7 @@ module ActiveRecordShards
     module_function
 
     def explode(conf)
-      conf = conf.deep_dup
+      conf = conf.to_h.deep_dup
 
       conf.to_a.each do |env_name, env_config|
         next unless shards = env_config.delete('shards')
@@ -23,9 +24,9 @@ module ActiveRecordShards
       end
 
       conf.to_a.each do |env_name, env_config|
-        if slave_conf = env_config.delete('slave')
-          expand_child!(env_config, slave_conf)
-          conf["#{env_name}_slave"] = slave_conf
+        if replica_conf = env_config.delete('replica')
+          expand_child!(env_config, replica_conf)
+          conf["#{env_name}_replica"] = replica_conf
         end
       end
 
@@ -34,7 +35,7 @@ module ActiveRecordShards
 
     def expand_child!(parent, child)
       parent.each do |key, value|
-        unless ['slave', 'shards'].include?(key) || value.is_a?(Hash)
+        unless ['replica', 'shards'].include?(key) || value.is_a?(Hash)
           child[key] ||= value
         end
       end

data/lib/active_record_shards/{connection_switcher_5_1.rb → connection_switcher-5-1.rb}

@@ -1,5 +1,15 @@
 module ActiveRecordShards
-  module BaseConfig
+  module ConnectionSwitcher
+    def connection_specification_name
+      name = current_shard_selection.resolve_connection_name(sharded: is_sharded?, configurations: configurations)
+
+      unless configurations[name] || name == "primary"
+        raise ActiveRecord::AdapterNotSpecified, "No database defined by #{name} in your database config. (configurations: #{configurations.to_h.keys.inspect})"
+      end
+
+      name
+    end
+
     private
 
     def ensure_shard_connection

data/lib/active_record_shards/connection_switcher-6-0.rb

@@ -0,0 +1,30 @@
+module ActiveRecordShards
+  module ConnectionSwitcher
+    def connection_specification_name
+      name = current_shard_selection.resolve_connection_name(sharded: is_sharded?, configurations: configurations)
+
+      @_ars_connection_specification_names ||= {}
+      unless @_ars_connection_specification_names.include?(name)
+        unless configurations[name] || name == "primary"
+          raise ActiveRecord::AdapterNotSpecified, "No database defined by #{name} in your database config. (configurations: #{configurations.to_h.keys.inspect})"
+        end
+
+        @_ars_connection_specification_names[name] = true
+      end
+
+      name
+    end
+
+    private
+
+    def ensure_shard_connection
+      # See if we've connected before. If not, call `#establish_connection`
+      # so that ActiveRecord can resolve connection_specification_name to an
+      # ARS connection.
+      spec_name = connection_specification_name
+
+      pool = connection_handler.retrieve_connection_pool(spec_name)
+      connection_handler.establish_connection(spec_name.to_sym) if pool.nil?
+    end
+  end
+end

data/lib/active_record_shards/connection_switcher.rb

@@ -1,9 +1,14 @@
 # frozen_string_literal: true
+
 require 'active_record_shards/shard_support'
 
 module ActiveRecordShards
   module ConnectionSwitcher
-    SHARD_NAMES_CONFIG_KEY = 'shard_names'.freeze
+    if "#{ActiveRecord::VERSION::MAJOR}.#{ActiveRecord::VERSION::MINOR}" == '6.1'
+      SHARD_NAMES_CONFIG_KEY = :shard_names
+    else
+      SHARD_NAMES_CONFIG_KEY = 'shard_names'
+    end
 
     def self.extended(base)
       base.singleton_class.send(:alias_method, :load_schema_without_default_shard!, :load_schema!)
@@ -13,17 +18,21 @@ module ActiveRecordShards
       base.singleton_class.send(:alias_method, :table_exists?, :table_exists_with_default_shard?)
     end
 
+    def on_primary_db(&block)
+      on_shard(nil, &block)
+    end
+
     def on_shard(shard)
-      old_selection = current_shard_selection
-      switch_connection(ShardSelection.new(shard)) if supports_sharding?
+      old_options = current_shard_selection.options
+      switch_connection(shard: shard) if supports_sharding?
       yield
     ensure
-      switch_connection(old_selection)
+      switch_connection(old_options)
     end
 
-    def on_first_shard
+    def on_first_shard(&block)
       shard_name = shard_names.first
-      on_shard(shard_name) { yield }
+      on_shard(shard_name, &block)
     end
 
     def shards
@@ -31,85 +40,144 @@ module ActiveRecordShards
     end
 
     def on_all_shards
-      old_selection = current_shard_selection
+      old_options = current_shard_selection.options
       if supports_sharding?
         shard_names.map do |shard|
-          switch_connection(ShardSelection.new(shard))
+          switch_connection(shard: shard)
           yield(shard)
         end
       else
         [yield]
       end
     ensure
-      switch_connection(old_selection)
+      switch_connection(old_options)
+    end
+
+    def on_replica_if(condition, &block)
+      condition ? on_replica(&block) : yield
+    end
+
+    def on_replica_unless(condition, &block)
+      on_replica_if(!condition, &block)
+    end
+
+    def on_primary_if(condition, &block)
+      condition ? on_primary(&block) : yield
     end
 
-    def connection_config
-      super.merge(current_shard_selection.connection_config.merge(sharded: is_sharded?))
+    def on_primary_unless(condition, &block)
+      on_primary_if(!condition, &block)
+    end
+
+    def on_primary_or_replica(which, &block)
+      if block_given?
+        on_cx_switch_block(which, &block)
+      else
+        PrimaryReplicaProxy.new(self, which)
+      end
+    end
+
+    # Executes queries using the replica database. Fails over to primary if no replica is found.
+    # if you want to execute a block of code on the replica you can go:
+    #   Account.on_replica do
+    #     Account.first
+    #   end
+    # the first account will be found on the replica DB
+    #
+    # For one-liners you can simply do
+    #   Account.on_replica.first
+    def on_replica(&block)
+      on_primary_or_replica(:replica, &block)
+    end
+
+    def on_primary(&block)
+      on_primary_or_replica(:primary, &block)
+    end
+
+    def on_cx_switch_block(which, force: false, construct_ro_scope: nil, &block)
+      @disallow_replica ||= 0
+      @disallow_replica += 1 if which == :primary
+
+      switch_to_replica = force || @disallow_replica.zero?
+      old_options = current_shard_selection.options
+
+      switch_connection(replica: switch_to_replica)
+
+      # we avoid_readonly_scope to prevent some stack overflow problems, like when
+      # .columns calls .with_scope which calls .columns and onward, endlessly.
+      if self == ActiveRecord::Base || !switch_to_replica || construct_ro_scope == false || ActiveRecordShards.disable_replica_readonly_records == true
+        yield
+      else
+        readonly.scoping(&block)
+      end
+    ensure
+      @disallow_replica -= 1 if which == :primary
+      switch_connection(old_options) if old_options
     end
 
     def supports_sharding?
       shard_names.any?
     end
 
-    def current_shard_selection
-      Thread.current[:shard_selection] ||= NoShardSelection.new
+    def on_replica?
+      current_shard_selection.on_replica?
     end
 
-    def current_shard_selection=(shard_selection)
-      Thread.current[:shard_selection] = shard_selection
+    def current_shard_selection
+      Thread.current[:shard_selection] ||= ShardSelection.new
     end
 
     def current_shard_id
       current_shard_selection.shard
     end
 
-    def on_shard?
-      current_shard_selection.on_shard?
-    end
-
     def shard_names
-      unless config = configurations[shard_env]
-        raise "Did not find #{shard_env} in configurations, did you forget to add it to your database config? (configurations: #{configurations.inspect})"
+      unless config_for_env.fetch(SHARD_NAMES_CONFIG_KEY, []).all? { |shard_name| shard_name.is_a?(Integer) }
+        raise "All shard names must be integers: #{config_for_env[SHARD_NAMES_CONFIG_KEY].inspect}."
       end
-      unless config[SHARD_NAMES_CONFIG_KEY]
-        raise "No shards configured for #{shard_env}"
-      end
-      unless config[SHARD_NAMES_CONFIG_KEY].all? { |shard_name| shard_name.is_a?(Integer) }
-        raise "All shard names must be integers: #{config[SHARD_NAMES_CONFIG_KEY].inspect}."
-      end
-      config[SHARD_NAMES_CONFIG_KEY]
-    end
 
-    def table_exists_with_default_shard?
-      with_default_shard { table_exists_without_default_shard? }
+      config_for_env[SHARD_NAMES_CONFIG_KEY] || []
     end
 
     private
 
-    def switch_connection(selection)
-      if selection.is_a?(ShardSelection)
+    def config_for_env
+      @_ars_config_for_env ||= {}
+      @_ars_config_for_env[shard_env] ||= begin
         unless config = configurations[shard_env]
-          raise "Did not find #{shard_env} in configurations, did you forget to add it to your database config? (configurations: #{configurations.inspect})"
+          raise "Did not find #{shard_env} in configurations, did you forget to add it to your database config? (configurations: #{configurations.to_h.keys.inspect})"
+        end
+
+        config
+      end
+    end
+    alias_method :check_config_for_env, :config_for_env
+
+    def switch_connection(options)
+      if options.any?
+        if options.key?(:replica)
+          current_shard_selection.on_replica = options[:replica]
         end
-        unless config['shard_names'].include?(selection.shard)
-          raise "Did not find shard #{selection.shard} in configurations"
+
+        if options.key?(:shard)
+          check_config_for_env
+
+          current_shard_selection.shard = options[:shard]
         end
-        self.current_shard_selection = selection
 
         ensure_shard_connection
-      else
-        self.current_shard_selection = selection
       end
     end
 
     def shard_env
-      ActiveRecordShards.rails_env
+      ActiveRecordShards.app_env
     end
 
-    def with_default_shard
-      if is_sharded? && !on_shard? && table_name != ActiveRecord::SchemaMigration.table_name
-        on_first_shard { yield }
+    # Make these few schema related methods available before having switched to
+    # a shard.
+    def with_default_shard(&block)
+      if is_sharded? && current_shard_id.nil? && table_name != ActiveRecord::SchemaMigration.table_name
+        on_first_shard(&block)
       else
         yield
       end
@@ -118,14 +186,30 @@ module ActiveRecordShards
     def load_schema_with_default_shard!
       with_default_shard { load_schema_without_default_shard! }
     end
+
+    def table_exists_with_default_shard?
+      with_default_shard { table_exists_without_default_shard? }
+    end
+
+    class PrimaryReplicaProxy
+      def initialize(target, which)
+        @target = target
+        @which = which
+      end
+
+      def method_missing(method, *args, &block) # rubocop:disable Style/MethodMissingSuper, Style/MissingRespondToMissing
+        @target.on_primary_or_replica(@which) { @target.send(method, *args, &block) }
+      end
+      ruby2_keywords(:method_missing) if respond_to?(:ruby2_keywords, true)
+    end
   end
 end
 
 case "#{ActiveRecord::VERSION::MAJOR}.#{ActiveRecord::VERSION::MINOR}"
-when '5.0'
-  require 'active_record_shards/connection_switcher_5_0'
 when '5.1', '5.2'
-  require 'active_record_shards/connection_switcher_5_1'
+  require 'active_record_shards/connection_switcher-5-1'
+when '6.0', '6.1'
+  require 'active_record_shards/connection_switcher-6-0'
 else
   raise "ActiveRecordShards is not compatible with #{ActiveRecord::VERSION::STRING}"
 end