switchman 3.0.5 → 4.0.0

data/lib/switchman/engine.rb

@@ -5,104 +5,23 @@ module Switchman
     isolate_namespace Switchman
 
     # enable Rails 6.1 style connection handling
-    config.active_record.legacy_connection_handling = false
+    config.active_record.legacy_connection_handling = false if ::Rails.version < "7.1"
     config.active_record.writing_role = :primary
 
-    config.autoload_once_paths << File.expand_path('app/models', config.paths.path)
+    ::GuardRail.singleton_class.prepend(GuardRail::ClassMethods)
 
-    def self.lookup_stores(cache_store_config)
-      result = {}
-      cache_store_config.each do |key, value|
-        next if value.is_a?(String)
-
-        result[key] = ::ActiveSupport::Cache.lookup_store(value)
-      end
-
-      cache_store_config.each do |key, value| # rubocop:disable Style/CombinableLoops
-        next unless value.is_a?(String)
-
-        result[key] = result[value]
-      end
-      result
-    end
-
-    initializer 'switchman.initialize_cache', before: 'initialize_cache' do
-      require 'switchman/active_support/cache'
-      ::ActiveSupport::Cache.singleton_class.prepend(ActiveSupport::Cache::ClassMethods)
-
-      # if we haven't already setup our cache map out-of-band, set it up from
-      # config.cache_store now. behaves similarly to Rails' default
-      # initialize_cache initializer, but for each value in the map, rather
-      # than just Rails.cache. if config.cache_store is a flat value, uses it
-      # to fill just the Rails.env entry in the cache map.
-      unless Switchman.config[:cache_map].present?
-        cache_store_config = ::Rails.configuration.cache_store
-        cache_store_config = { ::Rails.env => cache_store_config } unless cache_store_config.is_a?(Hash)
-
-        Switchman.config[:cache_map] = Engine.lookup_stores(cache_store_config)
-      end
-
-      # if the configured cache map (either from before, or as populated from
-      # config.cache_store) didn't have an entry for Rails.env, add one using
-      # lookup_store(nil); matches the behavior of Rails' default
-      # initialize_cache initializer when config.cache_store is nil.
-      unless Switchman.config[:cache_map].key?(::Rails.env)
-        value = ::ActiveSupport::Cache.lookup_store(nil)
-        Switchman.config[:cache_map][::Rails.env] = value
-      end
-
-      middlewares = Switchman.config[:cache_map].values.map do |store|
-        store.middleware if store.respond_to?(:middleware)
-      end.compact.uniq
-      middlewares.each do |middleware|
-        config.middleware.insert_before('Rack::Runtime', middleware)
-      end
-
-      # prevent :initialize_cache from trying to (or needing to) set
-      # Rails.cache. once our switchman.extend_ar initializer (below) runs
-      # Rails.cache will be overridden to pull appropriate values from the
-      # cache map, but between now and then, Rails.cache should return the
-      # Rails.env entry in the cache map.
-      ::Rails.cache = Switchman.config[:cache_map][::Rails.env]
-    end
-
-    initializer 'switchman.extend_ar', before: 'active_record.initialize_database' do
+    initializer "switchman.active_record_patch",
+                before: "active_record.initialize_database",
+                after: :setup_once_autoloader do
       ::ActiveSupport.on_load(:active_record) do
-        require 'switchman/active_record/abstract_adapter'
-        require 'switchman/active_record/association'
-        require 'switchman/active_record/attribute_methods'
-        require 'switchman/active_record/base'
-        require 'switchman/active_record/calculations'
-        require 'switchman/active_record/connection_pool'
-        require 'switchman/active_record/database_configurations'
-        require 'switchman/active_record/database_configurations/database_config'
-        require 'switchman/active_record/finder_methods'
-        require 'switchman/active_record/log_subscriber'
-        require 'switchman/active_record/migration'
-        require 'switchman/active_record/model_schema'
-        require 'switchman/active_record/persistence'
-        require 'switchman/active_record/predicate_builder'
-        require 'switchman/active_record/query_cache'
-        require 'switchman/active_record/query_methods'
-        require 'switchman/active_record/reflection'
-        require 'switchman/active_record/relation'
-        require 'switchman/active_record/spawn_methods'
-        require 'switchman/active_record/statement_cache'
-        require 'switchman/active_record/tasks/database_tasks'
-        require 'switchman/active_record/type_caster'
-        require 'switchman/arel'
-        require 'switchman/call_super'
-        require 'switchman/rails'
-        require 'switchman/guard_rail/relation'
-        require 'switchman/standard_error'
-
-        ::StandardError.include(StandardError)
+        # Switchman requires postgres, so just always load the pg adapter
+        require "active_record/connection_adapters/postgresql_adapter"
 
         self.default_shard = ::Rails.env.to_sym
         self.default_role = :primary
 
-        include ActiveRecord::Base
-        include ActiveRecord::AttributeMethods
+        prepend ActiveRecord::Base
+        prepend ActiveRecord::AttributeMethods
         include ActiveRecord::Persistence
         singleton_class.prepend ActiveRecord::ModelSchema::ClassMethods
 
@@ -111,25 +30,34 @@ module Switchman
         ::ActiveRecord::StatementCache::BindMap.prepend(ActiveRecord::StatementCache::BindMap)
         ::ActiveRecord::StatementCache::Substitute.send(:attr_accessor, :primary, :sharded)
 
-        ::ActiveRecord::Associations::CollectionAssociation.prepend(ActiveRecord::CollectionAssociation)
-        ::ActiveRecord::Associations::HasOneAssociation.prepend(ActiveRecord::ForeignAssociation)
-        ::ActiveRecord::Associations::HasManyAssociation.prepend(ActiveRecord::ForeignAssociation)
+        ::ActiveRecord::Associations::CollectionAssociation.prepend(ActiveRecord::Associations::CollectionAssociation)
+        ::ActiveRecord::Associations::HasOneAssociation.prepend(ActiveRecord::Associations::ForeignAssociation)
+        ::ActiveRecord::Associations::HasManyAssociation.prepend(ActiveRecord::Associations::ForeignAssociation)
 
         ::ActiveRecord::PredicateBuilder.singleton_class.prepend(ActiveRecord::PredicateBuilder)
 
-        prepend(ActiveRecord::AutosaveAssociation)
+        prepend(ActiveRecord::Associations::AutosaveAssociation)
 
-        ::ActiveRecord::Associations::Association.prepend(ActiveRecord::Association)
-        ::ActiveRecord::Associations::BelongsToAssociation.prepend(ActiveRecord::BelongsToAssociation)
-        ::ActiveRecord::Associations::CollectionProxy.include(ActiveRecord::CollectionProxy)
+        ::ActiveRecord::Associations::Association.prepend(ActiveRecord::Associations::Association)
+        ::ActiveRecord::Associations::BelongsToAssociation.prepend(ActiveRecord::Associations::BelongsToAssociation)
+        ::ActiveRecord::Associations::CollectionProxy.include(ActiveRecord::Associations::CollectionProxy)
 
-        ::ActiveRecord::Associations::Preloader::Association.prepend(ActiveRecord::Preloader::Association)
+        ::ActiveRecord::Associations::Preloader::Association.prepend(ActiveRecord::Associations::Preloader::Association)
+        ::ActiveRecord::Associations::Preloader::Association::LoaderRecords.prepend(
+          ActiveRecord::Associations::Preloader::Association::LoaderRecords
+        )
         ::ActiveRecord::ConnectionAdapters::AbstractAdapter.prepend(ActiveRecord::AbstractAdapter)
+        unless ::Rails.version < "7.1"
+          ::ActiveRecord::ConnectionAdapters::ConnectionHandler.prepend(ActiveRecord::ConnectionHandler)
+        end
         ::ActiveRecord::ConnectionAdapters::ConnectionPool.prepend(ActiveRecord::ConnectionPool)
         ::ActiveRecord::ConnectionAdapters::AbstractAdapter.prepend(ActiveRecord::QueryCache)
+        ::ActiveRecord::ConnectionAdapters::PostgreSQLAdapter.prepend(ActiveRecord::PostgreSQLAdapter)
 
         ::ActiveRecord::DatabaseConfigurations.prepend(ActiveRecord::DatabaseConfigurations)
-        ::ActiveRecord::DatabaseConfigurations::DatabaseConfig.prepend(ActiveRecord::DatabaseConfigurations::DatabaseConfig)
+        ::ActiveRecord::DatabaseConfigurations::DatabaseConfig.prepend(
+          ActiveRecord::DatabaseConfigurations::DatabaseConfig
+        )
 
         ::ActiveRecord::LogSubscriber.prepend(ActiveRecord::LogSubscriber)
         ::ActiveRecord::Migration.prepend(ActiveRecord::Migration)
@@ -137,6 +65,11 @@ module Switchman
         ::ActiveRecord::MigrationContext.prepend(ActiveRecord::MigrationContext)
         ::ActiveRecord::Migrator.prepend(ActiveRecord::Migrator)
 
+        if ::Rails.version > "7.1.3"
+          ::ActiveRecord::PendingMigrationConnection.singleton_class
+                                                    .include(ActiveRecord::PendingMigrationConnection::ClassMethods)
+        end
+
         ::ActiveRecord::Reflection::AbstractReflection.include(ActiveRecord::Reflection::AbstractReflection)
         ::ActiveRecord::Reflection::AssociationReflection.prepend(ActiveRecord::Reflection::AssociationScopeCache)
         ::ActiveRecord::Reflection::ThroughReflection.prepend(ActiveRecord::Reflection::AssociationScopeCache)
@@ -149,62 +82,77 @@ module Switchman
         ::ActiveRecord::Relation.include(ActiveRecord::SpawnMethods)
         ::ActiveRecord::Relation.include(CallSuper)
 
-        ::ActiveRecord::PredicateBuilder::AssociationQueryValue.prepend(ActiveRecord::PredicateBuilder::AssociationQueryValue)
-        ::ActiveRecord::PredicateBuilder::PolymorphicArrayValue.prepend(ActiveRecord::PredicateBuilder::AssociationQueryValue)
+        ::ActiveRecord::PredicateBuilder::PolymorphicArrayValue.prepend(
+          ActiveRecord::PredicateBuilder::PolymorphicArrayValue
+        )
 
         ::ActiveRecord::Tasks::DatabaseTasks.singleton_class.prepend(ActiveRecord::Tasks::DatabaseTasks)
 
+        ::ActiveRecord::TestFixtures.prepend(ActiveRecord::TestFixtures)
+
         ::ActiveRecord::TypeCaster::Map.include(ActiveRecord::TypeCaster::Map)
         ::ActiveRecord::TypeCaster::Connection.include(ActiveRecord::TypeCaster::Connection)
 
-        ::Rails.singleton_class.prepend(Rails::ClassMethods)
-
         ::Arel::Table.prepend(Arel::Table)
         ::Arel::Visitors::ToSql.prepend(Arel::Visitors::ToSql)
-      end
-    end
-
-    def self.foreign_key_check(name, type, limit: nil)
-      puts "WARNING: All foreign keys need to be 8-byte integers. #{name} looks like a foreign key. If so, please add the option: `:limit => 8`" if name.to_s =~ /_id\z/ && type.to_s == 'integer' && limit.to_i < 8
-    end
 
-    initializer 'switchman.extend_connection_adapters', after: 'active_record.initialize_database' do
-      ::ActiveSupport.on_load(:active_record) do
         ::ActiveRecord::ConnectionAdapters::AbstractAdapter.descendants.each do |klass|
           klass.prepend(ActiveRecord::AbstractAdapter::ForeignKeyCheck)
         end
 
-        require 'switchman/active_record/table_definition'
         ::ActiveRecord::ConnectionAdapters::TableDefinition.prepend(ActiveRecord::TableDefinition)
-
-        if defined?(::ActiveRecord::ConnectionAdapters::PostgreSQLAdapter)
-          require 'switchman/active_record/postgresql_adapter'
-          ::ActiveRecord::ConnectionAdapters::PostgreSQLAdapter.prepend(ActiveRecord::PostgreSQLAdapter)
-        end
-
-        Shard.send(:initialize_sharding)
       end
+      # Ensure that ActiveRecord::Base is always loaded before any app-level
+      # initializers can go try to load Switchman::Shard or we get a loop
+      ::ActiveRecord::Base
     end
 
-    initializer 'switchman.eager_load' do
-      ::ActiveSupport.on_load(:before_eager_load) do
-        # This needs to be loaded before Switchman::Shard, otherwise it won't autoload it correctly
-        require 'active_record/base'
+    initializer "switchman.error_patch", after: "active_record.initialize_database" do
+      ::ActiveSupport.on_load(:active_record) do
+        ::StandardError.include(StandardError)
       end
     end
 
-    initializer 'switchman.extend_guard_rail', before: 'switchman.extend_ar' do
-      ::ActiveSupport.on_load(:active_record) do
-        require 'switchman/guard_rail'
+    initializer "switchman.initialize_cache", before: :initialize_cache, after: "active_record.initialize_database" do
+      ::ActiveSupport::Cache.singleton_class.prepend(ActiveSupport::Cache::ClassMethods)
+
+      # if we haven't already setup our cache map out-of-band, set it up from
+      # config.cache_store now. behaves similarly to Rails' default
+      # initialize_cache initializer, but for each value in the map, rather
+      # than just Rails.cache. if config.cache_store is a flat value, uses it
+      # to fill just the Rails.env entry in the cache map.
+      unless Switchman.config[:cache_map].present?
+        cache_store_config = ::Rails.configuration.cache_store
+        cache_store_config = { ::Rails.env => cache_store_config } unless cache_store_config.is_a?(Hash)
 
-        ::GuardRail.singleton_class.prepend(GuardRail::ClassMethods)
+        Switchman.config[:cache_map] = ::ActiveSupport::Cache.lookup_stores(cache_store_config)
       end
-    end
 
-    initializer 'switchman.extend_controller', after: 'guard_rail.extend_ar' do
-      ::ActiveSupport.on_load(:action_controller) do
-        require 'switchman/action_controller/caching'
+      # if the configured cache map (either from before, or as populated from
+      # config.cache_store) didn't have an entry for Rails.env, add one using
+      # lookup_store(nil); matches the behavior of Rails' default
+      # initialize_cache initializer when config.cache_store is nil.
+      unless Switchman.config[:cache_map].key?(::Rails.env)
+        value = ::ActiveSupport::Cache.lookup_store(nil)
+        Switchman.config[:cache_map][::Rails.env] = value
+      end
+
+      middlewares = Switchman.config[:cache_map].values.filter_map do |store|
+        store.middleware if store.respond_to?(:middleware)
+      end.uniq
+      middlewares.each do |middleware|
+        config.middleware.insert_before("Rack::Runtime", middleware)
+      end
 
+      # prevent :initialize_cache from trying to (or needing to) set
+      # Rails.cache. once our switchman.extend_ar initializer (below) runs
+      # Rails.cache will be overridden to pull appropriate values from the
+      # cache map, but between now and then, Rails.cache should return the
+      # Rails.env entry in the cache map.
+      ::Rails.cache = Switchman.config[:cache_map][::Rails.env]
+      ::Rails.singleton_class.prepend(Rails::ClassMethods)
+
+      ::ActiveSupport.on_load(:action_controller) do
         ::ActionController::Base.include(ActionController::Caching)
       end
     end

data/lib/switchman/environment.rb

@@ -1,10 +1,10 @@
 # frozen_string_literal: true
 
-require 'etc'
+require "etc"
 
 module Switchman
   class Environment
-    def self.cpu_count(nproc_bin = 'nproc')
+    def self.cpu_count(nproc_bin = "nproc")
       return Etc.nprocessors if Etc.respond_to?(:nprocessors)
 
       `#{nproc_bin}`.to_i

data/lib/switchman/errors.rb

@@ -1,7 +1,22 @@
 # frozen_string_literal: true
 
 module Switchman
-  class NonExistentShardError < RuntimeError; end
+  module Errors
+    class ManuallyCreatedShadowRecordError < RuntimeError
+      DEFAULT_MSG = "It looks like you're trying to manually create a shadow record. " \
+                    "Please use Switchman::ActiveRecord::Base#save_shadow_record instead."
 
-  class ParallelShardExecError < RuntimeError; end
+      def initialize(msg = DEFAULT_MSG)
+        super
+      end
+    end
+
+    class NonExistentShardError < RuntimeError; end
+
+    class ParallelShardExecError < RuntimeError; end
+
+    class ShadowRecordError < RuntimeError; end
+
+    class UnshardedTableError < RuntimeError; end
+  end
 end

data/lib/switchman/guard_rail/relation.rb

@@ -5,21 +5,18 @@ module Switchman
     module Relation
       def exec_queries(*args)
         if lock_value
-          db = Shard.current(connection_classes).database_server
-          return db.unguard { super } if ::GuardRail.environment != db.guard_rail_environment
+          db = Shard.current(connection_class_for_self).database_server
+          db.unguard { super }
+        else
+          super
         end
-        super
       end
 
       %w[update_all delete_all].each do |method|
         class_eval <<-RUBY, __FILE__, __LINE__ + 1
-          def #{method}(*args)
-            db = Shard.current(connection_classes).database_server
-            if ::GuardRail.environment != db.guard_rail_environment
-              db.unguard { super }
-            else
-              super
-            end
+          def #{method}(*args, **kwargs)
+            db = Shard.current(connection_class_for_self).database_server
+            db.unguard { super }
           end
         RUBY
       end

data/lib/switchman/guard_rail.rb

@@ -3,6 +3,11 @@
 module Switchman
   module GuardRail
     module ClassMethods
+      def environment
+        # no overrides so we get the global role, not the role for the default shard
+        ::ActiveRecord::Base.current_role(without_overrides: true)
+      end
+
       def activate(role)
         DatabaseServer.send(:reference_role, role)
         super

data/lib/switchman/parallel.rb

@@ -0,0 +1,68 @@
+# frozen_string_literal: true
+
+require "parallel"
+
+module Switchman
+  module Parallel
+    module UndumpableException
+      def initialize(original)
+        super
+        @active_shards = original.instance_variable_get(:@active_shards)
+        current_shard
+      end
+    end
+
+    class QuietExceptionWrapper
+      attr_accessor :name
+
+      def initialize(name, wrapper)
+        @name = name
+        @wrapper = wrapper
+      end
+
+      def exception
+        @wrapper.exception
+      end
+    end
+
+    class UndumpableResult
+      attr_reader :name
+
+      def initialize(result)
+        @name = result.inspect
+      end
+
+      def inspect
+        "#<UndumpableResult:#{name}>"
+      end
+    end
+
+    class ResultWrapper
+      attr_reader :result
+
+      def initialize(result)
+        @result =
+          begin
+            Marshal.dump(result) && result
+          rescue
+            UndumpableResult.new(result)
+          end
+      end
+    end
+
+    class TransformingIO
+      delegate_missing_to :@original_io
+
+      def initialize(transformer, original_io)
+        @transformer = transformer
+        @original_io = original_io
+      end
+
+      def puts(*args)
+        args.flatten.each { |arg| @original_io.puts @transformer.call(arg) }
+      end
+    end
+  end
+end
+
+Parallel::UndumpableException.prepend(Switchman::Parallel::UndumpableException)

data/lib/switchman/r_spec_helper.rb

@@ -1,6 +1,6 @@
 # frozen_string_literal: true
 
-require 'switchman/test_helper'
+require "switchman/test_helper"
 
 module Switchman
   # including this module in your specs will give you several shards to
@@ -34,9 +34,9 @@ module Switchman
         groups = group.class.descendant_filtered_examples.map(&:example_group).uniq
         next unless groups.any? { |descendant_group| RSpecHelper.included_in?(descendant_group) }
 
-        puts 'Setting up sharding for all specs...'
+        puts "Setting up sharding for all specs..."
         Shard.delete_all
-        Switchman.cache.delete('default_shard')
+        Switchman.cache.delete("default_shard")
 
         @@shard1, @@shard2 = TestHelper.recreate_persistent_test_shards
         @@default_shard = Shard.default
@@ -48,7 +48,7 @@ module Switchman
             @@shard1 = @@shard1.create_new_shard
             @@shard2 = @@shard2.create_new_shard
           rescue => e
-            warn 'Sharding setup FAILED!:'
+            warn "Sharding setup FAILED!:"
             while e
               warn "\n#{e}\n"
               warn e.backtrace
@@ -66,14 +66,17 @@ module Switchman
         # we'll re-persist in the group's `before :all`; we don't want them to exist
         # in the db before then
         Shard.delete_all
-        Switchman.cache.delete('default_shard')
+        Switchman.cache.delete("default_shard")
         Shard.default(reload: true)
-        puts 'Done!'
+        puts "Done!"
 
+        main_pid = Process.pid
         at_exit do
+          next unless main_pid == Process.pid
+
           # preserve rspec's exit status
           status = $!.is_a?(::SystemExit) ? $!.status : nil
-          puts 'Tearing down sharding for all specs'
+          puts "Tearing down sharding for all specs"
           @@shard1.database_server.destroy unless @@shard1.database_server == Shard.default.database_server
           unless @@keep_the_shards
             @@shard1.drop_database
@@ -92,7 +95,7 @@ module Switchman
         dup = @@default_shard.dup
         dup.id = @@default_shard.id
         dup.save!
-        Switchman.cache.delete('default_shard')
+        Switchman.cache.delete("default_shard")
         Shard.default(reload: true)
         dup = @@shard1.dup
         dup.id = @@shard1.id
@@ -104,47 +107,33 @@ module Switchman
       end
 
       klass.before do
-        raise 'Sharding did not set up correctly' if @@sharding_failed
+        raise "Sharding did not set up correctly" if @@sharding_failed
 
         Shard.clear_cache
         if use_transactional_tests
           Shard.default(reload: true)
           @shard1 = Shard.find(@shard1.id)
           @shard2 = Shard.find(@shard2.id)
-          shards = [@shard2]
-          shards << @shard1 unless @shard1.database_server == Shard.default.database_server
-          shards.each do |shard|
-            shard.activate do
-              ::ActiveRecord::Base.connection.begin_transaction joinable: false
-            end
-          end
         end
       end
 
       klass.after do
         next if @@sharding_failed
 
-        if use_transactional_tests
-          shards = [@shard2]
-          shards << @shard1 unless @shard1.database_server == Shard.default.database_server
-          shards.each do |shard|
-            shard.activate do
-              ::ActiveRecord::Base.connection.rollback_transaction if ::ActiveRecord::Base.connection.transaction_open?
-            end
-          end
-        end
         # clean up after specs
-        DatabaseServer.all.each do |ds|
+        DatabaseServer.each do |ds|
           if ds.fake? && ds != @shard2.database_server
             ds.shards.delete_all unless use_transactional_tests
             ds.destroy
           end
+          ds.remove_instance_variable(:@primary_shard_id) if ds.instance_variable_defined?(:@primary_shard_id)
         end
       end
 
       klass.after(:all) do
-        Shard.connection.update("TRUNCATE #{Shard.quoted_table_name} CASCADE")
-        Switchman.cache.delete('default_shard')
+        # Don't truncate because that can create some fun cross-connection lock contention
+        Shard.delete_all
+        Switchman.cache.delete("default_shard")
         Shard.default(reload: true)
       end
     end

data/lib/switchman/rails.rb

@@ -4,10 +4,7 @@ module Switchman
   module Rails
     module ClassMethods
       def self.prepended(klass)
-        # in Rails 4+, the Rails.cache= method was used during bootstrap to set
-        # Rails.cache(_without_sharding) to the value from the config file. but now
-        # that that's done (the bootstrap happened before this module is included
-        # into Rails), we want to make sure no one tries to assign to Rails.cache,
+        # we want to make sure no one tries to assign to Rails.cache,
         # because it would be wrong w.r.t. sharding.
         klass.send(:remove_method, :cache=)
       end