switchman 1.13.3 → 2.0.0

data/lib/switchman/active_record/query_cache.rb

@@ -1,122 +1,32 @@
 module Switchman
   module ActiveRecord
     module QueryCache
-      if ::Rails.version < '5.0.1'
-        # thread local accessors to replace @query_cache_enabled
-        def query_cache
-          thread_cache = Thread.current[:query_cache] ||= {}
-          thread_cache[self.object_id] ||= Hash.new { |h,sql| h[sql] = {} }
-        end
-
-        def query_cache_enabled
-          Thread.current[:query_cache_enabled]
-        end
-
-        def query_cache_enabled=(value)
-          Thread.current[:query_cache_enabled] = value
-        end
-
-        # basically wholesale repeat of the methods from the original (see
-        # https://github.com/rails/rails/blob/master/activerecord/lib/active_record/connection_adapters/abstract/query_cache.rb),
-        # but with self.query_cache_enabled and self.query_cache_enabled= instead
-        # of @query_cache_enabled.
-
-        def enable_query_cache!
-          self.query_cache_enabled = true
-        end
-
-        def disable_query_cache!
-          self.query_cache_enabled = false
-        end
-
-        def cache
-          old, self.query_cache_enabled = query_cache_enabled, true
-          yield
-        ensure
-          self.query_cache_enabled = old
-          clear_query_cache unless self.query_cache_enabled
-        end
-
-        def uncached
-          old, self.query_cache_enabled = query_cache_enabled, false
-          yield
-        ensure
-          self.query_cache_enabled = old
-        end
-
-        def clear_query_cache
-          Thread.current[:query_cache]&.clear
-        end
-
-        def select_all(arel, name = nil, binds = [], preparable: nil)
-          if self.query_cache_enabled && !locked?(arel)
-            arel, binds = binds_from_relation(arel, binds)
-            sql = to_sql(arel, binds)
-            cache_sql(sql, binds) { super(sql, name, binds, preparable: preparable) }
-          else
-            super
-          end
-        end
-
-        # no reason to define these on the including class directly. the super
-        # works just as well from a method on the included module
-        [:insert, :update, :delete].each do |method_name|
-          class_eval <<-end_code, __FILE__, __LINE__ + 1
-            def #{method_name}(*args)
-              clear_query_cache if self.query_cache_enabled
-              super
-            end
-          end_code
-        end
-      end
 
       private
 
-      if ::Rails.version < '5.1'
-        def cache_sql(sql, binds)
-          # have to include the shard id in the cache key because of switching dbs on the same connection
-          sql = "#{self.shard.id}::#{sql}"
+      def cache_sql(sql, name, binds)
+        # have to include the shard id in the cache key because of switching dbs on the same connection
+        sql = "#{self.shard.id}::#{sql}"
+        @lock.synchronize do
           result =
               if query_cache[sql].key?(binds)
-                args = {:sql => sql, :binds => binds, :name => "CACHE", :connection_id => object_id}
-                args[:type_casted_binds] = -> { type_casted_binds(binds) } if ::Rails.version >= '5.0.7'
-                ::ActiveSupport::Notifications.instrument("sql.active_record", args)
+                args = {
+                  sql: sql,
+                  binds: binds,
+                  name: name,
+                  connection_id: object_id,
+                  cached: true
+                }
+                args[:type_casted_binds] = -> { type_casted_binds(binds) } if ::Rails.version >= '5.1.5'
+                ::ActiveSupport::Notifications.instrument(
+                    "sql.active_record",
+                    args
+                )
                 query_cache[sql][binds]
               else
                 query_cache[sql][binds] = yield
               end
-
-          if ::ActiveRecord::Result === result
-            result.dup
-          else
-            result.collect { |row| row.dup }
-          end
-        end
-      else
-        def cache_sql(sql, name, binds)
-          # have to include the shard id in the cache key because of switching dbs on the same connection
-          sql = "#{self.shard.id}::#{sql}"
-          @lock.synchronize do
-            result =
-                if query_cache[sql].key?(binds)
-                  args = {
-                    sql: sql,
-                    binds: binds,
-                    name: name,
-                    connection_id: object_id,
-                    cached: true
-                  }
-                  args[:type_casted_binds] = -> { type_casted_binds(binds) } if ::Rails.version >= '5.1.5'
-                  ::ActiveSupport::Notifications.instrument(
-                      "sql.active_record",
-                      args
-                  )
-                  query_cache[sql][binds]
-                else
-                  query_cache[sql][binds] = yield
-                end
-            result.dup
-          end
+          result.dup
         end
       end
     end

data/lib/switchman/active_record/query_methods.rb

@@ -227,6 +227,10 @@ module Switchman
         connection.with_local_table_name { super }
       end
 
+      def arel_column(columns)
+        connection.with_local_table_name { super }
+      end
+
       # semi-private
       public
       def transpose_predicates(predicates,

data/lib/switchman/active_record/relation.rb

@@ -5,7 +5,7 @@ module Switchman
         klass::SINGLE_VALUE_METHODS.concat [ :shard, :shard_source ]
       end
 
-      def initialize(*args)
+      def initialize(*, **)
         super
         self.shard_value = Shard.current(klass ? klass.shard_category : :primary) unless shard_value
         self.shard_source_value = :implicit unless shard_source_value
@@ -17,7 +17,7 @@ module Switchman
         result
       end
 
-      def merge(*args)
+      def merge(*)
         relation = super
         if relation.shard_value != self.shard_value && relation.shard_source_value == :implicit
           relation.shard_value = self.shard_value
@@ -26,15 +26,15 @@ module Switchman
         relation
       end
 
-      def new(*args, &block)
+      def new(*, &block)
         primary_shard.activate(klass.shard_category) { super }
       end
 
-      def create(*args, &block)
+      def create(*, &block)
         primary_shard.activate(klass.shard_category) { super }
       end
 
-      def create!(*args, &block)
+      def create!(*, &block)
         primary_shard.activate(klass.shard_category) { super }
       end
 

data/lib/switchman/active_record/statement_cache.rb

@@ -2,7 +2,7 @@ module Switchman
   module ActiveRecord
     module StatementCache
       module ClassMethods
-        def create(connection, block = Proc.new)
+        def create(connection, &block)
           relation = block.call ::ActiveRecord::StatementCache::Params.new
 
           if ::Rails.version >= "5.2"
@@ -46,37 +46,16 @@ module Switchman
         bind_values = bind_map.bind(params, current_shard, target_shard)
 
         target_shard.activate(klass.shard_category) do
-          if connection.use_qualified_names?
-            sql = qualified_query_builder(target_shard, klass).sql_for(bind_values, connection)
-            klass.find_by_sql(sql, bind_values)
-          else
-            sql = generic_query_builder(connection).sql_for(bind_values, connection)
-            klass.find_by_sql(sql, bind_values)
-          end
+          sql = qualified_query_builder(target_shard, klass).sql_for(bind_values, connection)
+          klass.find_by_sql(sql, bind_values)
         end
       end
 
-      if ::Rails.version < '5.1'
-        def generic_query_builder(connection)
-          @query_builder ||= connection.cacheable_query(@arel)
-        end
-
-        def qualified_query_builder(shard, klass)
-          @qualified_query_builders[shard.id] ||= klass.connection.cacheable_query(@arel)
-        end
-      elsif ::Rails.version < '5.2'
-        def generic_query_builder(connection)
-          @query_builder ||= connection.cacheable_query(self.class, @arel)
-        end
-
+      if ::Rails.version < '5.2'
         def qualified_query_builder(shard, klass)
           @qualified_query_builders[shard.id] ||= klass.connection.cacheable_query(self.class, @arel)
         end
       else
-        def generic_query_builder(connection)
-          @query_builder ||= connection.cacheable_query(self.class, @arel).first
-        end
-
         def qualified_query_builder(shard, klass)
           @qualified_query_builders[shard.id] ||= klass.connection.cacheable_query(self.class, @arel).first
         end

data/lib/switchman/active_record/table_definition.rb

@@ -1,8 +1,8 @@
 module Switchman
   module ActiveRecord
     module TableDefinition
-      def column(name, type, options = {})
-        Engine.foreign_key_check(name, type, options)
+      def column(name, type, limit: nil, **)
+        Engine.foreign_key_check(name, type, limit: limit)
         super
       end
     end

data/lib/switchman/active_support/cache.rb

@@ -4,10 +4,26 @@ module Switchman
       module ClassMethods
         def lookup_store(*store_options)
           store = super
+          # can't use defined?, because it's a _ruby_ autoloaded constant,
+          # so just checking that will cause it to get required
+          if store.class.name == "ActiveSupport::Cache::RedisCacheStore" && !::ActiveSupport::Cache::RedisCacheStore.ancestors.include?(RedisCacheStore)
+            ::ActiveSupport::Cache::RedisCacheStore.prepend(RedisCacheStore)
+          end
           store.options[:namespace] ||= lambda { Shard.current.default? ? nil : "shard_#{Shard.current.id}" }
           store
         end
       end
+
+      module RedisCacheStore
+        def clear(options = {})
+          # RedisCacheStore tries to be smart and only clear the cache under your namespace, if you have one set
+          # unfortunately, it uses the keys command, which is extraordinarily inefficient in a large redis instance
+          # fortunately, we can assume we control the entire instance, because we set up the namespacing, so just
+          # always unset it temporarily for clear calls
+          options[:namespace] = nil
+          super
+        end
+      end
     end
   end
 end

data/lib/switchman/connection_pool_proxy.rb

@@ -23,20 +23,27 @@ module Switchman
       @category = category
       @default_pool = default_pool
       @connection_pools = shard_connection_pools
-      @schema_cache = SchemaCache.new(self)
+      @schema_cache = default_pool.get_schema_cache(nil) if ::Rails.version >= '6'
+      @schema_cache = SchemaCache.new(self) unless @schema_cache.is_a?(SchemaCache)
+      if ::Rails.version >= '6'
+        @default_pool.set_schema_cache(@schema_cache)
+        @connection_pools.each_value do |pool|
+          pool.set_schema_cache(@schema_cache)
+        end
+      end
     end
 
     def active_shard
       Shard.current(@category)
     end
 
-    def active_shackles_environment
-      ::Rails.env.test? ? :master : active_shard.database_server.shackles_environment
+    def active_guard_rail_environment
+      ::Rails.env.test? ? :primary : active_shard.database_server.guard_rail_environment
     end
 
     def current_pool
       current_active_shard = active_shard
-      pool = self.default_pool if current_active_shard.database_server == Shard.default.database_server && active_shackles_environment == :master && (current_active_shard.default? || current_active_shard.database_server.shareable?)
+      pool = self.default_pool if current_active_shard.database_server == Shard.default.database_server && active_guard_rail_environment == :primary && (current_active_shard.default? || current_active_shard.database_server.shareable?)
       pool = @connection_pools[pool_key] ||= create_pool unless pool
       pool.shard = current_active_shard
       pool
@@ -46,21 +53,21 @@ module Switchman
       connection_pools.map(&:connections).inject([], &:+)
     end
 
-    def connection
+    def connection(switch_shard: true)
       pool = current_pool
       begin
-        connection = pool.connection
-        connection.instance_variable_set(:@schema_cache, @schema_cache)
+        connection = pool.connection(switch_shard: switch_shard)
+        connection.instance_variable_set(:@schema_cache, @schema_cache) unless ::Rails.version >= '6'
         connection
       rescue ConnectionError
-        raise if active_shard.database_server == Shard.default.database_server && active_shackles_environment == :master
-        configs = active_shard.database_server.config(active_shackles_environment)
+        raise if active_shard.database_server == Shard.default.database_server && active_guard_rail_environment == :primary
+        configs = active_shard.database_server.config(active_guard_rail_environment)
         raise unless configs.is_a?(Array)
         configs.each_with_index do |config, idx|
           pool = create_pool(config.dup)
           begin
             connection = pool.connection
-            connection.instance_variable_set(:@schema_cache, @schema_cache)
+            connection.instance_variable_set(:@schema_cache, @schema_cache) unless ::Rails.version >= '6'
           rescue ConnectionError
             raise if idx == configs.length - 1
             next
@@ -71,6 +78,10 @@ module Switchman
       end
     end
 
+    def get_schema_cache(_connection)
+      @schema_cache
+    end
+
     %w{release_connection
        disconnect!
        flush!
@@ -98,6 +109,10 @@ module Switchman
       connection_pools.each { |pool| pool.clear_idle_connections!(since_when) }
     end
 
+    def remove_shard!(shard)
+      connection_pools.each { |pool| pool.remove_shard!(shard) }
+    end
+
     protected
 
     def connection_pools
@@ -105,7 +120,7 @@ module Switchman
     end
 
     def pool_key
-      [active_shackles_environment,
+      [active_guard_rail_environment,
         active_shard.database_server.shareable? ? active_shard.database_server.pool_key : active_shard]
     end
 
@@ -113,7 +128,7 @@ module Switchman
       shard = active_shard
       unless config
         if shard != Shard.default
-          config = shard.database_server.config(active_shackles_environment)
+          config = shard.database_server.config(active_guard_rail_environment)
           config = config.first if config.is_a?(Array)
           config = config.dup
         else
@@ -123,27 +138,28 @@ module Switchman
           # different models could be using different configs on the default
           # shard, and database server wouldn't know about that
           config = default_pool.spec.instance_variable_get(:@config)
-          if config[active_shackles_environment].is_a?(Hash)
-            config = config.merge(config[active_shackles_environment])
-          elsif config[active_shackles_environment].is_a?(Array)
-            config = config.merge(config[active_shackles_environment].first)
+          if config[active_guard_rail_environment].is_a?(Hash)
+            config = config.merge(config[active_guard_rail_environment])
+          elsif config[active_guard_rail_environment].is_a?(Array)
+            config = config.merge(config[active_guard_rail_environment].first)
           else
             config = config.dup
           end
         end
       end
-      args = [config, "#{config[:adapter]}_connection"]
-      args.unshift(pool_key.join("/"))
-      spec = ::ActiveRecord::ConnectionAdapters::ConnectionSpecification.new(*args)
+      spec = ::ActiveRecord::ConnectionAdapters::ConnectionSpecification.new(
+        category,
+        config,
+        "#{config[:adapter]}_connection"
+      )
       # unfortunately the AR code that does this require logic can't really be
       # called in isolation
       require "active_record/connection_adapters/#{config[:adapter]}_adapter"
 
       ::ActiveRecord::ConnectionAdapters::ConnectionPool.new(spec).tap do |pool|
         pool.shard = shard
-        if ::Rails.version >= '5.0.1'
-          pool.enable_query_cache! if !@connection_pools.empty? && @connection_pools.first.last.query_cache_enabled
-        end
+        pool.set_schema_cache(@schema_cache) if ::Rails.version >= '6'
+        pool.enable_query_cache! if !@connection_pools.empty? && @connection_pools.first.last.query_cache_enabled
       end
     end
   end

data/lib/switchman/database_server.rb

@@ -40,8 +40,14 @@ module Switchman
       def database_servers
         unless @database_servers
           @database_servers = {}.with_indifferent_access
-          ::ActiveRecord::Base.configurations.each do |(id, config)|
-            @database_servers[id] = DatabaseServer.new(id, config)
+          if ::Rails.version >= '6.0'
+            ::ActiveRecord::Base.configurations.configurations.each do |config|
+              @database_servers[config.env_name] = DatabaseServer.new(config.env_name, config.config)
+            end
+          else
+            ::ActiveRecord::Base.configurations.each do |(id, config)|
+              @database_servers[id] = DatabaseServer.new(id, config)
+            end
           end
         end
         @database_servers
@@ -63,12 +69,12 @@ module Switchman
       @fake
     end
 
-    def config(environment = :master)
+    def config(environment = :primary)
       @configs[environment] ||= begin
         if @config[environment].is_a?(Array)
           @config[environment].map do |config|
             config = @config.merge((config || {}).symbolize_keys)
-            # make sure Shackles doesn't get any brilliant ideas about choosing the first possible server
+            # make sure GuardRail doesn't get any brilliant ideas about choosing the first possible server
             config.delete(environment)
             config
           end
@@ -80,33 +86,33 @@ module Switchman
       end
     end
 
-    def shackles_environment
-      @shackles_environment || ::Shackles.environment
+    def guard_rail_environment
+      @guard_rail_environment || ::GuardRail.environment
     end
 
     # locks this db to a specific environment, except for
     # when doing writes (then it falls back to the current
-    # value of Shackles.environment)
-    def shackle!(environment = :slave)
-      @shackles_environment = environment
+    # value of GuardRail.environment)
+    def guard!(environment = :secondary)
+      @guard_rail_environment = environment
     end
 
-    def unshackle!
-      @shackles_environment = nil
+    def unguard!
+      @guard_rail_environment = nil
     end
 
-    def unshackle
-      old_env = @shackles_environment
-      unshackle!
+    def unguard
+      old_env = @guard_rail_environment
+      unguard!
       yield
     ensure
-      shackle!(old_env)
+      guard!(old_env)
     end
 
     def shareable?
       @shareable_environment_key ||= []
-      environment = shackles_environment
-      explicit_user = ::Shackles.global_config[:username]
+      environment = guard_rail_environment
+      explicit_user = ::GuardRail.global_config[:username]
       return @shareable if @shareable_environment_key == [environment, explicit_user]
       @shareable_environment_key = [environment, explicit_user]
       if explicit_user
@@ -179,13 +185,18 @@ module Switchman
         shard = Shard.create!(:id => shard_id,
                               :name => name,
                               :database_server => self)
+        schema_already_existed = false
 
         begin
           self.class.creating_new_shard = true
           shard.activate(*Shard.categories) do
-            ::Shackles.activate(:deploy) do
+            ::GuardRail.activate(:deploy) do
               begin
                 if create_statement
+                  if (::ActiveRecord::Base.connection.select_value("SELECT 1 FROM pg_namespace WHERE nspname=#{::ActiveRecord::Base.connection.quote(name)}"))
+                    schema_already_existed = true
+                    raise "This schema already exists; cannot overwrite"
+                  end
                   Array(create_statement.call).each do |stmt|
                     ::ActiveRecord::Base.connection.execute(stmt)
                   end
@@ -223,7 +234,9 @@ module Switchman
           shard
         rescue
           shard.destroy
-          shard.drop_database rescue nil
+          unless schema_already_existed
+            shard.drop_database rescue nil
+          end
           reset_column_information unless create_schema == false rescue nil
           raise
         ensure

data/lib/switchman/default_shard.rb

@@ -3,6 +3,7 @@ require 'switchman/database_server'
 module Switchman
   class DefaultShard
     def id; 'default'; end
+    alias cache_key id
     def activate(*categories); yield; end
     def activate!(*categories); end
     def default?; true; end

data/lib/switchman/engine.rb

@@ -87,8 +87,8 @@ module Switchman
         require "switchman/arel"
         require "switchman/call_super"
         require "switchman/rails"
-        require "switchman/shackles/relation"
-        require_dependency "switchman/shard_internal"
+        require "switchman/guard_rail/relation"
+        require_dependency "switchman/shard"
         require "switchman/standard_error"
 
         ::StandardError.include(StandardError)
@@ -118,15 +118,11 @@ module Switchman
         ::ActiveRecord::ConnectionAdapters::ConnectionHandler.prepend(ActiveRecord::ConnectionHandler)
         ::ActiveRecord::ConnectionAdapters::ConnectionPool.prepend(ActiveRecord::ConnectionPool)
         ::ActiveRecord::ConnectionAdapters::AbstractAdapter.prepend(ActiveRecord::QueryCache)
-        # when we call super in Switchman::ActiveRecord::QueryCache#select_all,
-        # we want it to find the definition from
-        # ActiveRecord::ConnectionAdapters::DatabaseStatements, not
-        # ActiveRecord::ConnectionAdapters::QueryCache
-        ::ActiveRecord::ConnectionAdapters::QueryCache.send(:remove_method, :select_all) if ::Rails.version < '5.0.1'
 
         ::ActiveRecord::LogSubscriber.prepend(ActiveRecord::LogSubscriber)
         ::ActiveRecord::Migration.prepend(ActiveRecord::Migration)
         ::ActiveRecord::Migration::Compatibility::V5_0.prepend(ActiveRecord::Migration::Compatibility::V5_0)
+        ::ActiveRecord::MigrationContext.prepend(ActiveRecord::MigrationContext) if ::Rails.version >= '5.2'
         ::ActiveRecord::Migrator.prepend(ActiveRecord::Migrator)
 
         ::ActiveRecord::Reflection::AbstractReflection.include(ActiveRecord::Reflection::AbstractReflection)
@@ -137,7 +133,7 @@ module Switchman
         ::ActiveRecord::Relation.prepend(ActiveRecord::Calculations)
         ::ActiveRecord::Relation.include(ActiveRecord::FinderMethods)
         ::ActiveRecord::Relation.include(ActiveRecord::QueryMethods)
-        ::ActiveRecord::Relation.prepend(Shackles::Relation)
+        ::ActiveRecord::Relation.prepend(GuardRail::Relation)
         ::ActiveRecord::Relation.prepend(ActiveRecord::Relation)
         ::ActiveRecord::Relation.include(ActiveRecord::SpawnMethods)
         ::ActiveRecord::Relation.include(CallSuper)
@@ -154,8 +150,8 @@ module Switchman
       end
     end
 
-    def self.foreign_key_check(name, type, options)
-      if name.to_s =~ /_id\z/ && type.to_s == 'integer' && options[:limit].to_i < 8
+    def self.foreign_key_check(name, type, limit: nil)
+      if name.to_s =~ /_id\z/ && type.to_s == 'integer' && limit.to_i < 8
         puts "WARNING: All foreign keys need to be 8-byte integers. #{name} looks like a foreign key. If so, please add the option: `:limit => 8`"
       end
     end
@@ -173,6 +169,12 @@ module Switchman
           require "switchman/active_record/postgresql_adapter"
           ::ActiveRecord::ConnectionAdapters::PostgreSQLAdapter.prepend(ActiveRecord::PostgreSQLAdapter)
         end
+
+        # If Switchman::Shard wasn't loaded as of when ActiveRecord::Base initialized
+        # establish a connection here instead
+        if !Shard.instance_variable_get(:@default)
+          ::ActiveRecord::Base.establish_connection
+        end
       end
     end
 
@@ -183,15 +185,15 @@ module Switchman
       end
     end
 
-    initializer 'switchman.extend_shackles', :before => "switchman.extend_ar" do
+    initializer 'switchman.extend_guard_rail', :before => "switchman.extend_ar" do
       ::ActiveSupport.on_load(:active_record) do
-        require "switchman/shackles"
+        require "switchman/guard_rail"
 
-        ::Shackles.singleton_class.prepend(Shackles::ClassMethods)
+        ::GuardRail.singleton_class.prepend(GuardRail::ClassMethods)
       end
     end
 
-    initializer 'switchman.extend_controller', :after => "shackles.extend_ar" do
+    initializer 'switchman.extend_controller', :after => "guard_rail.extend_ar" do
       ::ActiveSupport.on_load(:action_controller) do
         require "switchman/action_controller/caching"
 

data/lib/switchman/{shackles → guard_rail}/relation.rb

@@ -1,11 +1,11 @@
 module Switchman
-  module Shackles
+  module GuardRail
     module Relation
       def exec_queries(*args)
         if self.lock_value
           db = Shard.current(shard_category).database_server
-          if ::Shackles.environment != db.shackles_environment
-            return db.unshackle { super }
+          if ::GuardRail.environment != db.guard_rail_environment
+            return db.unguard { super }
           end
         end
         super
@@ -15,8 +15,8 @@ module Switchman
         class_eval <<-RUBY, __FILE__, __LINE__ + 1
           def #{method}(*args)
             db = Shard.current(shard_category).database_server
-            if ::Shackles.environment != db.shackles_environment
-              db.unshackle { super }
+            if ::GuardRail.environment != db.guard_rail_environment
+              db.unguard { super }
             else
               super
             end

data/lib/switchman/{shackles.rb → guard_rail.rb}

@@ -1,17 +1,17 @@
 module Switchman
-  module Shackles
+  module GuardRail
     module ClassMethods
       def self.prepended(klass)
         klass.send(:remove_method, :ensure_handler)
       end
 
       # drops the save_handler and ensure_handler calls from the vanilla
-      # Shackles' implementation.
+      # GuardRail' implementation.
       def activate!(environment)
-        environment ||= :master
+        environment ||= :primary
         activated_environments << environment
         old_environment = self.environment
-        Thread.current[:shackles_environment] = environment
+        Thread.current[:guard_rail_environment] = environment
         old_environment
       end