switchman 3.0.14 → 3.5.20

data/lib/switchman/active_record/base.rb

@@ -14,8 +14,6 @@ module Switchman
         def sharded_model
           self.abstract_class = true
 
-          return if self == UnshardedRecord
-
           Shard.send(:add_sharded_model, self)
         end
 
@@ -24,24 +22,20 @@ module Switchman
           @integral_id
         end
 
-        def transaction(**)
-          if self != ::ActiveRecord::Base && current_scope
-            current_scope.activate do
-              db = Shard.current(connection_classes).database_server
-              if ::GuardRail.environment == db.guard_rail_environment
-                super
+        %w[transaction insert_all upsert_all].each do |method|
+          class_eval <<-RUBY, __FILE__, __LINE__ + 1
+            def #{method}(*, **)
+              if self != ::ActiveRecord::Base && current_scope
+                current_scope.activate do
+                  db = Shard.current(connection_class_for_self).database_server
+                  db.unguard { super }
+                end
               else
+                db = Shard.current(connection_class_for_self).database_server
                 db.unguard { super }
               end
             end
-          else
-            db = Shard.current(connection_classes).database_server
-            if ::GuardRail.environment == db.guard_rail_environment
-              super
-            else
-              db.unguard { super }
-            end
-          end
+          RUBY
         end
 
         def reset_column_information
@@ -63,16 +57,78 @@ module Switchman
         end
 
         def clear_query_caches_for_current_thread
-          ::ActiveRecord::Base.connection_handler.connection_pool_list.each do |pool|
+          pools = if ::Rails.version < "7.1"
+                    ::ActiveRecord::Base.connection_handler.connection_pool_list
+                  else
+                    ::ActiveRecord::Base.connection_handler.connection_pool_list(:all)
+                  end
+          pools.each do |pool|
             pool.connection(switch_shard: false).clear_query_cache if pool.active_connection?
           end
         end
 
+        def role_overriden?(shard_id)
+          current_role(target_shard: shard_id) != current_role(without_overrides: true)
+        end
+
+        def establish_connection(config_or_env = nil)
+          if config_or_env.is_a?(Symbol) && config_or_env != ::Rails.env.to_sym
+            raise ArgumentError,
+                  "establish connection cannot be used on the non-current shard/role"
+          end
+
+          # Ensure we don't randomly surprise change the connection parms associated with a shard/role
+          config_or_env = nil if config_or_env == ::Rails.env.to_sym
+
+          config_or_env ||= if current_shard == ::Rails.env.to_sym && current_role == :primary
+                              :primary
+                            else
+                              "#{current_shard}/#{current_role}".to_sym
+                            end
+
+          super(config_or_env)
+        end
+
+        def connected_to_stack
+          has_own_stack = if ::Rails.version < "7.0"
+                            Thread.current.thread_variable?(:ar_connected_to_stack)
+                          else
+                            ::ActiveSupport::IsolatedExecutionState.key?(:active_record_connected_to_stack)
+                          end
+
+          ret = super
+          return ret if has_own_stack
+
+          DatabaseServer.guard_servers
+          ret
+        end
+
+        # significant change: Allow per-shard roles
+        def current_role(without_overrides: false, target_shard: current_shard)
+          return super() if without_overrides
+
+          sharded_role = nil
+          connected_to_stack.reverse_each do |hash|
+            shard_role = hash.dig(:shard_roles, target_shard)
+            unless shard_role &&
+                   (hash[:klasses].include?(::ActiveRecord::Base) || hash[:klasses].include?(connection_class_for_self))
+              next
+            end
+
+            sharded_role = shard_role
+            break
+          end
+          # Allow a shard-specific role to be reverted to regular inheritance
+          return sharded_role if sharded_role && sharded_role != :_switchman_inherit
+
+          super()
+        end
+
         # significant change: _don't_ check if klasses.include?(Base)
         # i.e. other sharded models don't inherit the current shard of Base
         def current_shard
           connected_to_stack.reverse_each do |hash|
-            return hash[:shard] if hash[:shard] && hash[:klasses].include?(connection_classes)
+            return hash[:shard] if hash[:shard] && hash[:klasses].include?(connection_class_for_self)
           end
 
           default_shard
@@ -80,53 +136,131 @@ module Switchman
 
         def current_switchman_shard
           connected_to_stack.reverse_each do |hash|
-            return hash[:switchman_shard] if hash[:switchman_shard] && hash[:klasses].include?(connection_classes)
+            if hash[:switchman_shard] && hash[:klasses].include?(connection_class_for_self)
+              return hash[:switchman_shard]
+            end
           end
 
           Shard.default
         end
+
+        if ::Rails.version < "7.0"
+          def connection_class_for_self
+            connection_classes
+          end
+        end
       end
 
       def self.prepended(klass)
         klass.singleton_class.prepend(ClassMethods)
+        klass.scope :non_shadow, ->(key = primary_key) { where("#{key}<=? AND #{key}>?", Shard::IDS_PER_SHARD, 0) }
+        klass.scope :shadow, ->(key = primary_key) { where("#{key}>?", Shard::IDS_PER_SHARD) }
       end
 
       def _run_initialize_callbacks
         @shard ||= if self.class.sharded_primary_key?
-                     Shard.shard_for(self[self.class.primary_key], Shard.current(self.class.connection_classes))
+                     Shard.shard_for(self[self.class.primary_key], Shard.current(self.class.connection_class_for_self))
                    else
-                     Shard.current(self.class.connection_classes)
+                     Shard.current(self.class.connection_class_for_self)
                    end
+
+        @loaded_from_shard ||= Shard.current(self.class.connection_class_for_self)
+        if shadow_record? && !Switchman.config[:writable_shadow_records]
+          @readonly = true
+          @readonly_from_shadow ||= true
+        end
         super
       end
 
+      def readonly!
+        @readonly_from_shadow = false
+        super
+      end
+
+      def shadow_record?
+        pkey = self[self.class.primary_key]
+        return false unless self.class.sharded_column?(self.class.primary_key) && pkey
+
+        pkey > Shard::IDS_PER_SHARD
+      end
+
+      def canonical?
+        !shadow_record?
+      end
+
+      def save_shadow_record(new_attrs: attributes, target_shard: Shard.current)
+        return if target_shard == shard
+
+        shadow_attrs = {}
+        new_attrs.each do |attr, value|
+          shadow_attrs[attr] = if self.class.sharded_column?(attr)
+                                 Shard.relative_id_for(value, shard, target_shard)
+                               else
+                                 value
+                               end
+        end
+        target_shard.activate do
+          self.class.upsert(shadow_attrs, unique_by: self.class.primary_key)
+        end
+      end
+
+      def destroy_shadow_records(target_shards: [Shard.current])
+        raise Errors::ShadowRecordError, "Cannot be called on a shadow record." if shadow_record?
+
+        unless self.class.sharded_column?(self.class.primary_key)
+          raise Errors::MethodUnsupportedForUnshardedTableError,
+                "Cannot be called on a record belonging to an unsharded table."
+        end
+
+        Array(target_shards).each do |target_shard|
+          next if target_shard == shard
+
+          target_shard.activate { self.class.where("id = ?", global_id).delete_all }
+        end
+      end
+
+      # Returns "the shard that this record was actually loaded from" , as
+      # opposed to "the shard this record belongs on", which might be
+      # different if this is a shadow record.
+      def loaded_from_shard
+        @loaded_from_shard || shard
+      end
+
       def shard
-        @shard || Shard.current(self.class.connection_classes) || Shard.default
+        @shard || fallback_shard
       end
 
       def shard=(new_shard)
         raise ::ActiveRecord::ReadOnlyRecord if !new_record? || @shard_set_in_stone
 
-        return if shard == new_shard
+        if shard == new_shard
+          @loaded_from_shard = new_shard
+          return
+        end
 
         attributes.each do |attr, value|
           self[attr] = Shard.relative_id_for(value, shard, new_shard) if self.class.sharded_column?(attr)
         end
+        @loaded_from_shard = new_shard
         @shard = new_shard
       end
 
       def save(*, **)
+        raise Errors::ManuallyCreatedShadowRecordError if creating_shadow_record?
+
         @shard_set_in_stone = true
         super
       end
 
       def save!(*, **)
+        raise Errors::ManuallyCreatedShadowRecordError if creating_shadow_record?
+
         @shard_set_in_stone = true
         super
       end
 
       def destroy
-        shard.activate(self.class.connection_classes) { super }
+        shard.activate(self.class.connection_class_for_self) { super }
       end
 
       def clone
@@ -138,20 +272,21 @@ module Switchman
         result
       end
 
-      def transaction(**kwargs, &block)
-        shard.activate(self.class.connection_classes) do
-          self.class.transaction(**kwargs, &block)
+      def transaction(...)
+        shard.activate(self.class.connection_class_for_self) do
+          self.class.transaction(...)
         end
       end
 
       def with_transaction_returning_status
-        shard.activate(self.class.connection_classes) do
-          super
+        shard.activate(self.class.connection_class_for_self) do
+          db = Shard.current(self.class.connection_class_for_self).database_server
+          db.unguard { super }
         end
       end
 
       def hash
-        self.class.sharded_primary_key? ? self.class.hash ^ global_id.hash : super
+        self.class.sharded_primary_key? ? [self.class, global_id].hash : super
       end
 
       def to_param
@@ -167,15 +302,15 @@ module Switchman
 
       def update_columns(*)
         db = shard.database_server
-        return db.unguard { super } if ::GuardRail.environment != db.guard_rail_environment
-
-        super
+        db.unguard { super }
       end
 
       def id_for_database
         if self.class.sharded_primary_key?
-          # It's an int, so so it's safe to just return it without passing it through anything else
-          # In theory we should do `@attributes[@primary_key].type.serialize(id)`, but that seems to have surprising side-effects
+          # It's an int, so it's safe to just return it without passing it
+          # through anything else. In theory we should do
+          # `@attributes[@primary_key].type.serialize(id)`, but that seems to
+          # have surprising side-effects
           id
         else
           super
@@ -184,24 +319,34 @@ module Switchman
 
       protected
 
-      # see also AttributeMethods#connection_classes_code_for_reflection
-      def connection_classes_for_reflection(reflection)
+      # see also AttributeMethods#connection_class_for_self_code_for_reflection
+      def connection_class_for_self_for_reflection(reflection)
         if reflection
           if reflection.options[:polymorphic]
             begin
-              read_attribute(reflection.foreign_type)&.constantize&.connection_classes || ::ActiveRecord::Base
+              read_attribute(reflection.foreign_type)&.constantize&.connection_class_for_self || ::ActiveRecord::Base
             rescue NameError
               # in case someone is abusing foreign_type to not point to an actual class
               ::ActiveRecord::Base
             end
           else
             # otherwise we can just return a symbol for the statically known type of the association
-            reflection.klass.connection_classes
+            reflection.klass.connection_class_for_self
           end
         else
-          self.class.connection_classes
+          self.class.connection_class_for_self
         end
       end
+
+      private
+
+      def fallback_shard
+        Shard.current(self.class.connection_class_for_self) || Shard.default
+      end
+
+      def creating_shadow_record?
+        new_record? && shadow_record?
+      end
     end
   end
 end

data/lib/switchman/active_record/calculations.rb

@@ -4,7 +4,7 @@ module Switchman
   module ActiveRecord
     module Calculations
       def pluck(*column_names)
-        target_shard = Shard.current(klass.connection_classes)
+        target_shard = Shard.current(klass.connection_class_for_self)
         shard_count = 0
         result = activate do |relation, shard|
           shard_count += 1
@@ -28,7 +28,7 @@ module Switchman
 
       def execute_simple_calculation(operation, column_name, distinct)
         operation = operation.to_s.downcase
-        if operation == 'average'
+        if operation == "average"
           result = calculate_simple_average(column_name, distinct)
         else
           result = activate do |relation|
@@ -36,11 +36,11 @@ module Switchman
           end
           if result.is_a?(Array)
             case operation
-            when 'count', 'sum'
+            when "count", "sum"
               result = result.sum
-            when 'minimum'
+            when "minimum"
               result = result.min
-            when 'maximum'
+            when "maximum"
               result = result.max
             end
           end
@@ -52,20 +52,20 @@ module Switchman
         # See activerecord#execute_simple_calculation
         relation = except(:order)
         column = aggregate_column(column_name)
-        relation.select_values = [operation_over_aggregate_column(column, 'average', distinct).as('average'),
-                                  operation_over_aggregate_column(column, 'count', distinct).as('count')]
+        relation.select_values = [operation_over_aggregate_column(column, "average", distinct).as("average"),
+                                  operation_over_aggregate_column(column, "count", distinct).as("count")]
 
         initial_results = relation.activate { |rel| klass.connection.select_all(rel) }
         if initial_results.is_a?(Array)
           initial_results.each do |r|
-            r['average'] = type_cast_calculated_value_switchman(r['average'], column_name, 'average')
-            r['count'] = type_cast_calculated_value_switchman(r['count'], column_name, 'count')
+            r["average"] = type_cast_calculated_value_switchman(r["average"], column_name, "average")
+            r["count"] = type_cast_calculated_value_switchman(r["count"], column_name, "count")
           end
-          result = initial_results.map { |r| r['average'] * r['count'] }.sum / initial_results.map do |r|
-                                                                                 r['count']
-                                                                               end.sum
+          result = initial_results.sum { |r| r["average"] * r["count"] } / initial_results.sum do |r|
+                                                                             r["count"]
+                                                                           end
         else
-          result = type_cast_calculated_value_switchman(initial_results.first['average'], column_name, 'average')
+          result = type_cast_calculated_value_switchman(initial_results.first["average"], column_name, "average")
         end
         result
       end
@@ -90,7 +90,7 @@ module Switchman
             row[opts[:aggregate_alias]] = type_cast_calculated_value_switchman(
               row[opts[:aggregate_alias]], column_name, opts[:operation]
             )
-            row['count'] = row['count'].to_i if opts[:operation] == 'average'
+            row["count"] = row["count"].to_i if opts[:operation] == "average"
 
             opts[:group_columns].each do |aliaz, _type, group_column_name|
               if opts[:associated] && (aliaz == opts[:group_aliases].first)
@@ -109,48 +109,71 @@ module Switchman
       private
 
       def type_cast_calculated_value_switchman(value, column_name, operation)
-        type_cast_calculated_value(value, operation) do |val|
+        if ::Rails.version < "7.0"
+          type_cast_calculated_value(value, operation) do |val|
+            column = aggregate_column(column_name)
+            type ||= column.try(:type_caster) ||
+                     lookup_cast_type_from_join_dependencies(column_name.to_s) || ::ActiveRecord::Type.default_value
+            type.deserialize(val)
+          end
+        else
           column = aggregate_column(column_name)
           type ||= column.try(:type_caster) ||
-                   lookup_cast_type_from_join_dependencies(column_name.to_s) || Type.default_value
-          type.deserialize(val)
+                   lookup_cast_type_from_join_dependencies(column_name.to_s) || ::ActiveRecord::Type.default_value
+          type_cast_calculated_value(value, operation, type)
         end
       end
 
       def column_name_for(field)
-        field.respond_to?(:name) ? field.name.to_s : field.to_s.split('.').last
+        field.respond_to?(:name) ? field.name.to_s : field.to_s.split(".").last
       end
 
       def grouped_calculation_options(operation, column_name, distinct)
         opts = { operation: operation, column_name: column_name, distinct: distinct }
 
-        opts[:aggregate_alias] = aggregate_alias_for(operation, column_name)
+        # Rails 7.0.5
+        if defined?(::ActiveRecord::Calculations::ColumnAliasTracker)
+          column_alias_tracker = ::ActiveRecord::Calculations::ColumnAliasTracker.new(connection)
+        end
+
+        opts[:aggregate_alias] = aggregate_alias_for(operation, column_name, column_alias_tracker)
         group_attrs = group_values
         if group_attrs.first.respond_to?(:to_sym)
           association  = klass.reflect_on_association(group_attrs.first.to_sym)
-          associated   = group_attrs.size == 1 && association && association.macro == :belongs_to # only count belongs_to associations
+          # only count belongs_to associations
+          associated   = group_attrs.size == 1 && association && association.macro == :belongs_to
           group_fields = Array(associated ? association.foreign_key : group_attrs)
         else
           group_fields = group_attrs
         end
 
-        # to_s is because Rails 5 returns a string but Rails 6 returns a symbol.
-        group_aliases = group_fields.map { |field| column_alias_for(field.downcase.to_s).to_s }
+        group_aliases = group_fields.map do |field|
+          field = connection.visitor.compile(field) if ::Arel.arel_node?(field)
+          if column_alias_tracker
+            column_alias_tracker.alias_for(field.to_s.downcase)
+          else
+            column_alias_for(field.to_s.downcase)
+          end
+        end
         group_columns = group_aliases.zip(group_fields).map do |aliaz, field|
           [aliaz, type_for(field), column_name_for(field)]
         end
-        opts.merge!(association: association, associated: associated,
-                    group_aliases: group_aliases, group_columns: group_columns,
+        opts.merge!(association: association,
+                    associated: associated,
+                    group_aliases: group_aliases,
+                    group_columns: group_columns,
                     group_fields: group_fields)
 
         opts
       end
 
-      def aggregate_alias_for(operation, column_name)
-        if operation == 'count' && column_name == :all
-          'count_all'
-        elsif operation == 'average'
-          'average'
+      def aggregate_alias_for(operation, column_name, column_alias_tracker)
+        if operation == "count" && column_name == :all
+          "count_all"
+        elsif operation == "average"
+          "average"
+        elsif column_alias_tracker
+          column_alias_tracker.alias_for("#{operation} #{column_name}")
         else
           column_alias_for("#{operation} #{column_name}")
         end
@@ -166,13 +189,14 @@ module Switchman
             opts[:distinct]
           ).as(opts[:aggregate_alias])
         ]
-        if opts[:operation] == 'average'
+        if opts[:operation] == "average"
           # include count in average so we can recalculate the average
           # across all shards if needed
           select_values << operation_over_aggregate_column(
             aggregate_column(opts[:column_name]),
-            'count', opts[:distinct]
-          ).as('count')
+            "count",
+            opts[:distinct]
+          ).as("count")
         end
 
         haves = having_clause.send(:predicates)
@@ -198,22 +222,22 @@ module Switchman
           key = key.first if key.size == 1
           value = row[opts[:aggregate_alias]]
 
-          if opts[:operation] == 'average'
+          if opts[:operation] == "average"
             if result.key?(key)
               old_value, old_count = result[key]
-              new_count = old_count + row['count']
-              new_value = ((old_value * old_count) + (value * row['count'])) / new_count
+              new_count = old_count + row["count"]
+              new_value = ((old_value * old_count) + (value * row["count"])) / new_count
               result[key] = [new_value, new_count]
             else
-              result[key] = [value, row['count']]
+              result[key] = [value, row["count"]]
             end
           elsif result.key?(key)
             case opts[:operation]
-            when 'count', 'sum'
+            when "count", "sum"
               result[key] += value
-            when 'minimum'
+            when "minimum"
               result[key] = value if value < result[key]
-            when 'maximum'
+            when "maximum"
               result[key] = value if value > result[key]
             end
           else
@@ -221,7 +245,7 @@ module Switchman
           end
         end
 
-        result.transform_values!(&:first) if opts[:operation] == 'average'
+        result.transform_values!(&:first) if opts[:operation] == "average"
 
         result
       end

data/lib/switchman/active_record/connection_handler.rb

@@ -0,0 +1,18 @@
+# frozen_string_literal: true
+
+module Switchman
+  module ActiveRecord
+    module ConnectionHandler
+      def resolve_pool_config(config, connection_name, role, shard)
+        ret = super
+        # Make *all* pool configs use the same schema reflection
+        ret.schema_reflection = ConnectionHandler.global_schema_reflection
+        ret
+      end
+
+      def self.global_schema_reflection
+        @global_schema_reflection ||= ::ActiveRecord::ConnectionAdapters::SchemaReflection.new(nil)
+      end
+    end
+  end
+end

data/lib/switchman/active_record/connection_pool.rb

@@ -1,10 +1,27 @@
 # frozen_string_literal: true
 
-require 'switchman/errors'
-
 module Switchman
   module ActiveRecord
     module ConnectionPool
+      if ::Rails.version < "7.1"
+        def get_schema_cache(connection)
+          self.schema_cache ||= SharedSchemaCache.get_schema_cache(connection)
+          self.schema_cache.connection = connection
+
+          self.schema_cache
+        end
+
+        # rubocop:disable Naming/AccessorMethodName override method
+        def set_schema_cache(cache)
+          schema_cache = get_schema_cache(cache.connection)
+
+          cache.instance_variables.each do |x|
+            schema_cache.instance_variable_set(x, cache.instance_variable_get(x))
+          end
+        end
+        # rubocop:enable Naming/AccessorMethodName override method
+      end
+
       def default_schema
         connection unless @schemas
         # default shard will not switch databases immediately, so it won't be set yet
@@ -20,7 +37,7 @@ module Switchman
 
       def connection(switch_shard: true)
         conn = super()
-        raise NonExistentShardError if current_shard.new_record?
+        raise Errors::NonExistentShardError if current_shard.new_record?
 
         switch_database(conn) if conn.shard != current_shard && switch_shard
         conn
@@ -33,7 +50,9 @@ module Switchman
       end
 
       def switch_database(conn)
-        @schemas = conn.current_schemas if !@schemas && conn.adapter_name == 'PostgreSQL' && !current_shard.database_server.config[:shard_name]
+        if !@schemas && conn.adapter_name == "PostgreSQL" && !current_shard.database_server.config[:shard_name]
+          @schemas = conn.current_schemas
+        end
 
         conn.shard = current_shard
       end
@@ -41,7 +60,7 @@ module Switchman
       private
 
       def current_shard
-        connection_klass.current_switchman_shard
+        (::Rails.version < "7.0") ? connection_klass.current_switchman_shard : connection_class.current_switchman_shard
       end
 
       def tls_key