switchman 3.2.1 → 4.0.0

data/lib/switchman/active_record/base.rb

@@ -22,13 +22,59 @@ module Switchman
           @integral_id
         end
 
-        %w[transaction insert_all upsert_all].each do |method|
+        def transaction(**)
+          if self != ::ActiveRecord::Base && current_scope
+            current_scope.activate do
+              db = Shard.current(connection_class_for_self).database_server
+              db.unguard { super }
+            end
+          else
+            db = Shard.current(connection_class_for_self).database_server
+            db.unguard { super }
+          end
+        end
+
+        # NOTE: `returning` values are _not_ transposed back to the current shard
+        %w[insert_all upsert_all].each do |method|
           class_eval <<-RUBY, __FILE__, __LINE__ + 1
-            def #{method}(*, **)
-              if self != ::ActiveRecord::Base && current_scope
-                current_scope.activate do
+            def #{method}(attributes, returning: nil, **)
+              scope = self != ::ActiveRecord::Base && current_scope
+              if (target_shard = scope&.primary_shard) == (current_shard = Shard.current(connection_class_for_self))
+                scope = nil
+              end
+              if scope
+                dupped = false
+                attributes.each_with_index do |hash, i|
+                  if dupped || hash.any? { |k, v| sharded_column?(k) }
+                    unless dupped
+                      attributes = attributes.dup
+                      dupped = true
+                    end
+                    attributes[i] = hash.to_h do |k, v|
+                      if sharded_column?(k)
+                        [k, Shard.relative_id_for(v, current_shard, target_shard)]
+                      else
+                        [k, v]
+                      end
+                    end
+                  end
+                end
+              end
+
+              if scope
+                scope.activate do
                   db = Shard.current(connection_class_for_self).database_server
-                  db.unguard { super }
+                  result = db.unguard { super }
+                  if result&.columns&.any? { |c| sharded_column?(c) }
+                    transposed_rows = result.rows.map do |row|
+                      row.map.with_index do |value, i|
+                        sharded_column?(result.columns[i]) ? Shard.relative_id_for(value, target_shard, current_shard) : value
+                      end
+                    end
+                    result = ::ActiveRecord::Result.new(result.columns, transposed_rows, result.column_types)
+                  end
+
+                  result
                 end
               else
                 db = Shard.current(connection_class_for_self).database_server
@@ -57,7 +103,12 @@ module Switchman
         end
 
         def clear_query_caches_for_current_thread
-          ::ActiveRecord::Base.connection_handler.connection_pool_list.each do |pool|
+          pools = if ::Rails.version < "7.1"
+                    ::ActiveRecord::Base.connection_handler.connection_pool_list
+                  else
+                    ::ActiveRecord::Base.connection_handler.connection_pool_list(:all)
+                  end
+          pools.each do |pool|
             pool.connection(switch_shard: false).clear_query_cache if pool.active_connection?
           end
         end
@@ -67,7 +118,10 @@ module Switchman
         end
 
         def establish_connection(config_or_env = nil)
-          raise ArgumentError, 'establish connection cannot be used on the non-current shard/role' if config_or_env.is_a?(Symbol) && config_or_env != ::Rails.env.to_sym
+          if config_or_env.is_a?(Symbol) && config_or_env != ::Rails.env.to_sym
+            raise ArgumentError,
+                  "establish connection cannot be used on the non-current shard/role"
+          end
 
           # Ensure we don't randomly surprise change the connection parms associated with a shard/role
           config_or_env = nil if config_or_env == ::Rails.env.to_sym
@@ -75,16 +129,18 @@ module Switchman
           config_or_env ||= if current_shard == ::Rails.env.to_sym && current_role == :primary
                               :primary
                             else
-                              "#{current_shard}/#{current_role}".to_sym
+                              :"#{current_shard}/#{current_role}"
                             end
 
           super(config_or_env)
         end
 
         def connected_to_stack
-          return super if ::Rails.version < '7.0' ? Thread.current.thread_variable?(:ar_connected_to_stack) : ::ActiveSupport::IsolatedExecutionState.key?(:active_record_connected_to_stack)
+          has_own_stack = ::ActiveSupport::IsolatedExecutionState.key?(:active_record_connected_to_stack)
 
           ret = super
+          return ret if has_own_stack
+
           DatabaseServer.guard_servers
           ret
         end
@@ -96,10 +152,13 @@ module Switchman
           sharded_role = nil
           connected_to_stack.reverse_each do |hash|
             shard_role = hash.dig(:shard_roles, target_shard)
-            if shard_role && (hash[:klasses].include?(::ActiveRecord::Base) || hash[:klasses].include?(connection_class_for_self))
-              sharded_role = shard_role
-              break
+            unless shard_role &&
+                   (hash[:klasses].include?(::ActiveRecord::Base) || hash[:klasses].include?(connection_class_for_self))
+              next
             end
+
+            sharded_role = shard_role
+            break
           end
           # Allow a shard-specific role to be reverted to regular inheritance
           return sharded_role if sharded_role && sharded_role != :_switchman_inherit
@@ -119,21 +178,24 @@ module Switchman
 
         def current_switchman_shard
           connected_to_stack.reverse_each do |hash|
-            return hash[:switchman_shard] if hash[:switchman_shard] && hash[:klasses].include?(connection_class_for_self)
+            if hash[:switchman_shard] && hash[:klasses].include?(connection_class_for_self)
+              return hash[:switchman_shard]
+            end
           end
 
           Shard.default
         end
-
-        if ::Rails.version < '7.0'
-          def connection_class_for_self
-            connection_classes
-          end
-        end
       end
 
       def self.prepended(klass)
         klass.singleton_class.prepend(ClassMethods)
+        klass.scope :non_shadow, lambda { |key = primary_key|
+                                   where(key => (QueryMethods::NonTransposingValue.new(0)..
+                                                 QueryMethods::NonTransposingValue.new(Shard::IDS_PER_SHARD)))
+                                 }
+        klass.scope :shadow, lambda { |key = primary_key|
+                               where(key => QueryMethods::NonTransposingValue.new(Shard::IDS_PER_SHARD)..)
+                             }
       end
 
       def _run_initialize_callbacks
@@ -142,7 +204,17 @@ module Switchman
                    else
                      Shard.current(self.class.connection_class_for_self)
                    end
-        readonly! if shadow_record? && !Switchman.config[:writable_shadow_records]
+
+        @loaded_from_shard ||= Shard.current(self.class.connection_class_for_self)
+        if shadow_record? && !Switchman.config[:writable_shadow_records]
+          @readonly = true
+          @readonly_from_shadow ||= true
+        end
+        super
+      end
+
+      def readonly!
+        @readonly_from_shadow = false
         super
       end
 
@@ -153,6 +225,10 @@ module Switchman
         pkey > Shard::IDS_PER_SHARD
       end
 
+      def canonical?
+        !shadow_record?
+      end
+
       def save_shadow_record(new_attrs: attributes, target_shard: Shard.current)
         return if target_shard == shard
 
@@ -169,27 +245,57 @@ module Switchman
         end
       end
 
+      def destroy_shadow_records(target_shards: [Shard.current])
+        raise Errors::ShadowRecordError, "Cannot be called on a shadow record." if shadow_record?
+
+        unless self.class.sharded_column?(self.class.primary_key)
+          raise Errors::MethodUnsupportedForUnshardedTableError,
+                "Cannot be called on a record belonging to an unsharded table."
+        end
+
+        Array(target_shards).each do |target_shard|
+          next if target_shard == shard
+
+          target_shard.activate { self.class.where("id = ?", global_id).delete_all }
+        end
+      end
+
+      # Returns "the shard that this record was actually loaded from" , as
+      # opposed to "the shard this record belongs on", which might be
+      # different if this is a shadow record.
+      def loaded_from_shard
+        @loaded_from_shard || shard
+      end
+
       def shard
-        @shard || Shard.current(self.class.connection_class_for_self) || Shard.default
+        @shard || fallback_shard
       end
 
       def shard=(new_shard)
         raise ::ActiveRecord::ReadOnlyRecord if !new_record? || @shard_set_in_stone
 
-        return if shard == new_shard
+        if shard == new_shard
+          @loaded_from_shard = new_shard
+          return
+        end
 
         attributes.each do |attr, value|
           self[attr] = Shard.relative_id_for(value, shard, new_shard) if self.class.sharded_column?(attr)
         end
+        @loaded_from_shard = new_shard
         @shard = new_shard
       end
 
       def save(*, **)
+        raise Errors::ManuallyCreatedShadowRecordError if creating_shadow_record?
+
         @shard_set_in_stone = true
         super
       end
 
       def save!(*, **)
+        raise Errors::ManuallyCreatedShadowRecordError if creating_shadow_record?
+
         @shard_set_in_stone = true
         super
       end
@@ -207,9 +313,9 @@ module Switchman
         result
       end
 
-      def transaction(**kwargs, &block)
+      def transaction(...)
         shard.activate(self.class.connection_class_for_self) do
-          self.class.transaction(**kwargs, &block)
+          self.class.transaction(...)
         end
       end
 
@@ -221,7 +327,7 @@ module Switchman
       end
 
       def hash
-        self.class.sharded_primary_key? ? self.class.hash ^ global_id.hash : super
+        self.class.sharded_primary_key? ? [self.class, global_id].hash : super
       end
 
       def to_param
@@ -242,8 +348,10 @@ module Switchman
 
       def id_for_database
         if self.class.sharded_primary_key?
-          # It's an int, so so it's safe to just return it without passing it through anything else
-          # In theory we should do `@attributes[@primary_key].type.serialize(id)`, but that seems to have surprising side-effects
+          # It's an int, so it's safe to just return it without passing it
+          # through anything else. In theory we should do
+          # `@attributes[@primary_key].type.serialize(id)`, but that seems to
+          # have surprising side-effects
           id
         else
           super
@@ -270,6 +378,16 @@ module Switchman
           self.class.connection_class_for_self
         end
       end
+
+      private
+
+      def fallback_shard
+        Shard.current(self.class.connection_class_for_self) || Shard.default
+      end
+
+      def creating_shadow_record?
+        new_record? && shadow_record?
+      end
     end
   end
 end

data/lib/switchman/active_record/calculations.rb

@@ -28,7 +28,7 @@ module Switchman
 
       def execute_simple_calculation(operation, column_name, distinct)
         operation = operation.to_s.downcase
-        if operation == 'average'
+        if operation == "average"
           result = calculate_simple_average(column_name, distinct)
         else
           result = activate do |relation|
@@ -36,11 +36,11 @@ module Switchman
           end
           if result.is_a?(Array)
             case operation
-            when 'count', 'sum'
+            when "count", "sum"
               result = result.sum
-            when 'minimum'
+            when "minimum"
               result = result.min
-            when 'maximum'
+            when "maximum"
               result = result.max
             end
           end
@@ -52,20 +52,20 @@ module Switchman
         # See activerecord#execute_simple_calculation
         relation = except(:order)
         column = aggregate_column(column_name)
-        relation.select_values = [operation_over_aggregate_column(column, 'average', distinct).as('average'),
-                                  operation_over_aggregate_column(column, 'count', distinct).as('count')]
+        relation.select_values = [operation_over_aggregate_column(column, "average", distinct).as("average"),
+                                  operation_over_aggregate_column(column, "count", distinct).as("count")]
 
         initial_results = relation.activate { |rel| klass.connection.select_all(rel) }
         if initial_results.is_a?(Array)
           initial_results.each do |r|
-            r['average'] = type_cast_calculated_value_switchman(r['average'], column_name, 'average')
-            r['count'] = type_cast_calculated_value_switchman(r['count'], column_name, 'count')
+            r["average"] = type_cast_calculated_value_switchman(r["average"], column_name, "average")
+            r["count"] = type_cast_calculated_value_switchman(r["count"], column_name, "count")
           end
-          result = initial_results.map { |r| r['average'] * r['count'] }.sum / initial_results.map do |r|
-                                                                                 r['count']
-                                                                               end.sum
+          result = initial_results.sum { |r| r["average"] * r["count"] } / initial_results.sum do |r|
+                                                                             r["count"]
+                                                                           end
         else
-          result = type_cast_calculated_value_switchman(initial_results.first['average'], column_name, 'average')
+          result = type_cast_calculated_value_switchman(initial_results.first["average"], column_name, "average")
         end
         result
       end
@@ -90,7 +90,7 @@ module Switchman
             row[opts[:aggregate_alias]] = type_cast_calculated_value_switchman(
               row[opts[:aggregate_alias]], column_name, opts[:operation]
             )
-            row['count'] = row['count'].to_i if opts[:operation] == 'average'
+            row["count"] = row["count"].to_i if opts[:operation] == "average"
 
             opts[:group_columns].each do |aliaz, _type, group_column_name|
               if opts[:associated] && (aliaz == opts[:group_aliases].first)
@@ -106,58 +106,104 @@ module Switchman
         compact_grouped_calculation_rows(rows, opts)
       end
 
-      private
+      if ::Rails.version >= "7.1"
+        def ids
+          return super unless klass.sharded_primary_key?
 
-      def type_cast_calculated_value_switchman(value, column_name, operation)
-        if ::Rails.version < '7.0'
-          type_cast_calculated_value(value, operation) do |val|
-            column = aggregate_column(column_name)
-            type ||= column.try(:type_caster) ||
-                     lookup_cast_type_from_join_dependencies(column_name.to_s) || ::ActiveRecord::Type.default_value
-            type.deserialize(val)
+          if loaded?
+            result = records.map do |record|
+              Shard.relative_id_for(record._read_attribute(primary_key),
+                                    record.shard,
+                                    Shard.current(klass.connection_class_for_self))
+            end
+            return @async ? Promise::Complete.new(result) : result
+          end
+
+          if has_include?(primary_key)
+            relation = apply_join_dependency.group(primary_key)
+            return relation.ids
+          end
+
+          columns = arel_columns([primary_key])
+          base_shard = Shard.current(klass.connection_class_for_self)
+          activate do |r|
+            relation = r.spawn
+            relation.select_values = columns
+
+            result = if relation.where_clause.contradiction?
+                       ::ActiveRecord::Result.empty
+                     else
+                       skip_query_cache_if_necessary do
+                         klass.connection.select_all(relation, "#{klass.name} Ids", async: @async)
+                       end
+                     end
+
+            result.then do |res|
+              type_cast_pluck_values(res, columns).map { |id| Shard.relative_id_for(id, Shard.current, base_shard) }
+            end
           end
-        else
-          column = aggregate_column(column_name)
-          type ||= column.try(:type_caster) ||
-                   lookup_cast_type_from_join_dependencies(column_name.to_s) || ::ActiveRecord::Type.default_value
-          type_cast_calculated_value(value, operation, type)
         end
       end
 
+      private
+
+      def type_cast_calculated_value_switchman(value, column_name, operation)
+        column = aggregate_column(column_name)
+        type ||= column.try(:type_caster) ||
+                 lookup_cast_type_from_join_dependencies(column_name.to_s) || ::ActiveRecord::Type.default_value
+        type_cast_calculated_value(value, operation, type)
+      end
+
       def column_name_for(field)
-        field.respond_to?(:name) ? field.name.to_s : field.to_s.split('.').last
+        field.respond_to?(:name) ? field.name.to_s : field.to_s.split(".").last
       end
 
       def grouped_calculation_options(operation, column_name, distinct)
         opts = { operation: operation, column_name: column_name, distinct: distinct }
 
-        opts[:aggregate_alias] = aggregate_alias_for(operation, column_name)
+        # Rails 7.0.5
+        if defined?(::ActiveRecord::Calculations::ColumnAliasTracker)
+          column_alias_tracker = ::ActiveRecord::Calculations::ColumnAliasTracker.new(connection)
+        end
+
+        opts[:aggregate_alias] = aggregate_alias_for(operation, column_name, column_alias_tracker)
         group_attrs = group_values
         if group_attrs.first.respond_to?(:to_sym)
           association  = klass.reflect_on_association(group_attrs.first.to_sym)
-          associated   = group_attrs.size == 1 && association && association.macro == :belongs_to # only count belongs_to associations
+          # only count belongs_to associations
+          associated   = group_attrs.size == 1 && association && association.macro == :belongs_to
           group_fields = Array(associated ? association.foreign_key : group_attrs)
         else
           group_fields = group_attrs
         end
 
-        # to_s is because Rails 5 returns a string but Rails 6 returns a symbol.
-        group_aliases = group_fields.map { |field| column_alias_for(field.downcase.to_s).to_s }
+        group_aliases = group_fields.map do |field|
+          field = connection.visitor.compile(field) if ::Arel.arel_node?(field)
+          if column_alias_tracker
+            column_alias_tracker.alias_for(field.to_s.downcase)
+          else
+            column_alias_for(field.to_s.downcase)
+          end
+        end
         group_columns = group_aliases.zip(group_fields).map do |aliaz, field|
           [aliaz, type_for(field), column_name_for(field)]
         end
-        opts.merge!(association: association, associated: associated,
-                    group_aliases: group_aliases, group_columns: group_columns,
+        opts.merge!(association: association,
+                    associated: associated,
+                    group_aliases: group_aliases,
+                    group_columns: group_columns,
                     group_fields: group_fields)
 
         opts
       end
 
-      def aggregate_alias_for(operation, column_name)
-        if operation == 'count' && column_name == :all
-          'count_all'
-        elsif operation == 'average'
-          'average'
+      def aggregate_alias_for(operation, column_name, column_alias_tracker)
+        if operation == "count" && column_name == :all
+          "count_all"
+        elsif operation == "average"
+          "average"
+        elsif column_alias_tracker
+          column_alias_tracker.alias_for("#{operation} #{column_name}")
         else
           column_alias_for("#{operation} #{column_name}")
         end
@@ -173,13 +219,14 @@ module Switchman
             opts[:distinct]
           ).as(opts[:aggregate_alias])
         ]
-        if opts[:operation] == 'average'
+        if opts[:operation] == "average"
           # include count in average so we can recalculate the average
           # across all shards if needed
           select_values << operation_over_aggregate_column(
             aggregate_column(opts[:column_name]),
-            'count', opts[:distinct]
-          ).as('count')
+            "count",
+            opts[:distinct]
+          ).as("count")
         end
 
         haves = having_clause.send(:predicates)
@@ -205,22 +252,22 @@ module Switchman
           key = key.first if key.size == 1
           value = row[opts[:aggregate_alias]]
 
-          if opts[:operation] == 'average'
+          if opts[:operation] == "average"
             if result.key?(key)
               old_value, old_count = result[key]
-              new_count = old_count + row['count']
-              new_value = ((old_value * old_count) + (value * row['count'])) / new_count
+              new_count = old_count + row["count"]
+              new_value = ((old_value * old_count) + (value * row["count"])) / new_count
               result[key] = [new_value, new_count]
             else
-              result[key] = [value, row['count']]
+              result[key] = [value, row["count"]]
             end
           elsif result.key?(key)
             case opts[:operation]
-            when 'count', 'sum'
+            when "count", "sum"
               result[key] += value
-            when 'minimum'
+            when "minimum"
               result[key] = value if value < result[key]
-            when 'maximum'
+            when "maximum"
               result[key] = value if value > result[key]
             end
           else
@@ -228,7 +275,7 @@ module Switchman
           end
         end
 
-        result.transform_values!(&:first) if opts[:operation] == 'average'
+        result.transform_values!(&:first) if opts[:operation] == "average"
 
         result
       end

data/lib/switchman/active_record/connection_handler.rb

@@ -0,0 +1,18 @@
+# frozen_string_literal: true
+
+module Switchman
+  module ActiveRecord
+    module ConnectionHandler
+      def resolve_pool_config(config, connection_name, role, shard)
+        ret = super
+        # Make *all* pool configs use the same schema reflection
+        ret.schema_reflection = ConnectionHandler.global_schema_reflection
+        ret
+      end
+
+      def self.global_schema_reflection
+        @global_schema_reflection ||= ::ActiveRecord::ConnectionAdapters::SchemaReflection.new(nil)
+      end
+    end
+  end
+end

data/lib/switchman/active_record/connection_pool.rb

@@ -3,6 +3,25 @@
 module Switchman
   module ActiveRecord
     module ConnectionPool
+      if ::Rails.version < "7.1"
+        def get_schema_cache(connection)
+          self.schema_cache ||= SharedSchemaCache.get_schema_cache(connection)
+          self.schema_cache.connection = connection
+
+          self.schema_cache
+        end
+
+        # rubocop:disable Naming/AccessorMethodName override method
+        def set_schema_cache(cache)
+          schema_cache = get_schema_cache(cache.connection)
+
+          cache.instance_variables.each do |x|
+            schema_cache.instance_variable_set(x, cache.instance_variable_get(x))
+          end
+        end
+        # rubocop:enable Naming/AccessorMethodName override method
+      end
+
       def default_schema
         connection unless @schemas
         # default shard will not switch databases immediately, so it won't be set yet
@@ -31,7 +50,9 @@ module Switchman
       end
 
       def switch_database(conn)
-        @schemas = conn.current_schemas if !@schemas && conn.adapter_name == 'PostgreSQL' && !current_shard.database_server.config[:shard_name]
+        if !@schemas && conn.adapter_name == "PostgreSQL" && !current_shard.database_server.config[:shard_name]
+          @schemas = conn.current_schemas
+        end
 
         conn.shard = current_shard
       end
@@ -39,11 +60,11 @@ module Switchman
       private
 
       def current_shard
-        ::Rails.version < '7.0' ? connection_klass.current_switchman_shard : connection_class.current_switchman_shard
+        connection_class.current_switchman_shard
       end
 
       def tls_key
-        "#{object_id}_shard".to_sym
+        :"#{object_id}_shard"
       end
     end
   end