switchman 3.4.2 → 3.6.7

data/lib/switchman/active_record/base.rb

@@ -22,13 +22,59 @@ module Switchman
           @integral_id
         end
 
-        %w[transaction insert_all upsert_all].each do |method|
+        def transaction(**)
+          if self != ::ActiveRecord::Base && current_scope
+            current_scope.activate do
+              db = Shard.current(connection_class_for_self).database_server
+              db.unguard { super }
+            end
+          else
+            db = Shard.current(connection_class_for_self).database_server
+            db.unguard { super }
+          end
+        end
+
+        # NOTE: `returning` values are _not_ transposed back to the current shard
+        %w[insert_all upsert_all].each do |method|
           class_eval <<-RUBY, __FILE__, __LINE__ + 1
-            def #{method}(*, **)
-              if self != ::ActiveRecord::Base && current_scope
-                current_scope.activate do
+            def #{method}(attributes, returning: nil, **)
+              scope = self != ::ActiveRecord::Base && current_scope
+              if (target_shard = scope&.primary_shard) == (current_shard = Shard.current(connection_class_for_self))
+                scope = nil
+              end
+              if scope
+                dupped = false
+                attributes.each_with_index do |hash, i|
+                  if dupped || hash.any? { |k, v| sharded_column?(k) }
+                    unless dupped
+                      attributes = attributes.dup
+                      dupped = true
+                    end
+                    attributes[i] = hash.to_h do |k, v|
+                      if sharded_column?(k)
+                        [k, Shard.relative_id_for(v, current_shard, target_shard)]
+                      else
+                        [k, v]
+                      end
+                    end
+                  end
+                end
+              end
+
+              if scope
+                scope.activate do
                   db = Shard.current(connection_class_for_self).database_server
-                  db.unguard { super }
+                  result = db.unguard { super }
+                  if result&.columns&.any? { |c| sharded_column?(c) }
+                    transposed_rows = result.rows.map do |row|
+                      row.map.with_index do |value, i|
+                        sharded_column?(result.columns[i]) ? Shard.relative_id_for(value, target_shard, current_shard) : value
+                      end
+                    end
+                    result = ::ActiveRecord::Result.new(result.columns, transposed_rows, result.column_types)
+                  end
+
+                  result
                 end
               else
                 db = Shard.current(connection_class_for_self).database_server
@@ -57,7 +103,12 @@ module Switchman
         end
 
         def clear_query_caches_for_current_thread
-          ::ActiveRecord::Base.connection_handler.connection_pool_list.each do |pool|
+          pools = if ::Rails.version < "7.1"
+                    ::ActiveRecord::Base.connection_handler.connection_pool_list
+                  else
+                    ::ActiveRecord::Base.connection_handler.connection_pool_list(:all)
+                  end
+          pools.each do |pool|
             pool.connection(switch_shard: false).clear_query_cache if pool.active_connection?
           end
         end
@@ -67,7 +118,10 @@ module Switchman
         end
 
         def establish_connection(config_or_env = nil)
-          raise ArgumentError, 'establish connection cannot be used on the non-current shard/role' if config_or_env.is_a?(Symbol) && config_or_env != ::Rails.env.to_sym
+          if config_or_env.is_a?(Symbol) && config_or_env != ::Rails.env.to_sym
+            raise ArgumentError,
+                  "establish connection cannot be used on the non-current shard/role"
+          end
 
           # Ensure we don't randomly surprise change the connection parms associated with a shard/role
           config_or_env = nil if config_or_env == ::Rails.env.to_sym
@@ -75,16 +129,22 @@ module Switchman
           config_or_env ||= if current_shard == ::Rails.env.to_sym && current_role == :primary
                               :primary
                             else
-                              "#{current_shard}/#{current_role}".to_sym
+                              :"#{current_shard}/#{current_role}"
                             end
 
           super(config_or_env)
         end
 
         def connected_to_stack
-          return super if ::Rails.version < '7.0' ? Thread.current.thread_variable?(:ar_connected_to_stack) : ::ActiveSupport::IsolatedExecutionState.key?(:active_record_connected_to_stack)
+          has_own_stack = if ::Rails.version < "7.0"
+                            Thread.current.thread_variable?(:ar_connected_to_stack)
+                          else
+                            ::ActiveSupport::IsolatedExecutionState.key?(:active_record_connected_to_stack)
+                          end
 
           ret = super
+          return ret if has_own_stack
+
           DatabaseServer.guard_servers
           ret
         end
@@ -96,10 +156,13 @@ module Switchman
           sharded_role = nil
           connected_to_stack.reverse_each do |hash|
             shard_role = hash.dig(:shard_roles, target_shard)
-            if shard_role && (hash[:klasses].include?(::ActiveRecord::Base) || hash[:klasses].include?(connection_class_for_self))
-              sharded_role = shard_role
-              break
+            unless shard_role &&
+                   (hash[:klasses].include?(::ActiveRecord::Base) || hash[:klasses].include?(connection_class_for_self))
+              next
             end
+
+            sharded_role = shard_role
+            break
           end
           # Allow a shard-specific role to be reverted to regular inheritance
           return sharded_role if sharded_role && sharded_role != :_switchman_inherit
@@ -119,13 +182,15 @@ module Switchman
 
         def current_switchman_shard
           connected_to_stack.reverse_each do |hash|
-            return hash[:switchman_shard] if hash[:switchman_shard] && hash[:klasses].include?(connection_class_for_self)
+            if hash[:switchman_shard] && hash[:klasses].include?(connection_class_for_self)
+              return hash[:switchman_shard]
+            end
           end
 
           Shard.default
         end
 
-        if ::Rails.version < '7.0'
+        if ::Rails.version < "7.0"
           def connection_class_for_self
             connection_classes
           end
@@ -134,6 +199,13 @@ module Switchman
 
       def self.prepended(klass)
         klass.singleton_class.prepend(ClassMethods)
+        klass.scope :non_shadow, lambda { |key = primary_key|
+                                   where(key => (QueryMethods::NonTransposingValue.new(0)..
+                                                 QueryMethods::NonTransposingValue.new(Shard::IDS_PER_SHARD)))
+                                 }
+        klass.scope :shadow, lambda { |key = primary_key|
+                               where(key => QueryMethods::NonTransposingValue.new(Shard::IDS_PER_SHARD)..)
+                             }
       end
 
       def _run_initialize_callbacks
@@ -144,7 +216,15 @@ module Switchman
                    end
 
         @loaded_from_shard ||= Shard.current(self.class.connection_class_for_self)
-        readonly! if shadow_record? && !Switchman.config[:writable_shadow_records]
+        if shadow_record? && !Switchman.config[:writable_shadow_records]
+          @readonly = true
+          @readonly_from_shadow ||= true
+        end
+        super
+      end
+
+      def readonly!
+        @readonly_from_shadow = false
         super
       end
 
@@ -155,6 +235,10 @@ module Switchman
         pkey > Shard::IDS_PER_SHARD
       end
 
+      def canonical?
+        !shadow_record?
+      end
+
       def save_shadow_record(new_attrs: attributes, target_shard: Shard.current)
         return if target_shard == shard
 
@@ -172,13 +256,17 @@ module Switchman
       end
 
       def destroy_shadow_records(target_shards: [Shard.current])
-        raise Errors::ShadowRecordError, 'Cannot be called on a shadow record.' if shadow_record?
-        raise Errors::MethodUnsupportedForUnshardedTableError, 'Cannot be called on a record belonging to an unsharded table.' unless self.class.sharded_column?(self.class.primary_key)
+        raise Errors::ShadowRecordError, "Cannot be called on a shadow record." if shadow_record?
+
+        unless self.class.sharded_column?(self.class.primary_key)
+          raise Errors::MethodUnsupportedForUnshardedTableError,
+                "Cannot be called on a record belonging to an unsharded table."
+        end
 
         Array(target_shards).each do |target_shard|
           next if target_shard == shard
 
-          target_shard.activate { self.class.where('id = ?', global_id).delete_all }
+          target_shard.activate { self.class.where("id = ?", global_id).delete_all }
         end
       end
 
@@ -235,9 +323,9 @@ module Switchman
         result
       end
 
-      def transaction(**kwargs, &block)
+      def transaction(...)
         shard.activate(self.class.connection_class_for_self) do
-          self.class.transaction(**kwargs, &block)
+          self.class.transaction(...)
         end
       end
 
@@ -270,8 +358,10 @@ module Switchman
 
       def id_for_database
         if self.class.sharded_primary_key?
-          # It's an int, so so it's safe to just return it without passing it through anything else
-          # In theory we should do `@attributes[@primary_key].type.serialize(id)`, but that seems to have surprising side-effects
+          # It's an int, so it's safe to just return it without passing it
+          # through anything else. In theory we should do
+          # `@attributes[@primary_key].type.serialize(id)`, but that seems to
+          # have surprising side-effects
           id
         else
           super

data/lib/switchman/active_record/calculations.rb

@@ -28,7 +28,7 @@ module Switchman
 
       def execute_simple_calculation(operation, column_name, distinct)
         operation = operation.to_s.downcase
-        if operation == 'average'
+        if operation == "average"
           result = calculate_simple_average(column_name, distinct)
         else
           result = activate do |relation|
@@ -36,11 +36,11 @@ module Switchman
           end
           if result.is_a?(Array)
             case operation
-            when 'count', 'sum'
+            when "count", "sum"
               result = result.sum
-            when 'minimum'
+            when "minimum"
               result = result.min
-            when 'maximum'
+            when "maximum"
               result = result.max
             end
           end
@@ -52,20 +52,20 @@ module Switchman
         # See activerecord#execute_simple_calculation
         relation = except(:order)
         column = aggregate_column(column_name)
-        relation.select_values = [operation_over_aggregate_column(column, 'average', distinct).as('average'),
-                                  operation_over_aggregate_column(column, 'count', distinct).as('count')]
+        relation.select_values = [operation_over_aggregate_column(column, "average", distinct).as("average"),
+                                  operation_over_aggregate_column(column, "count", distinct).as("count")]
 
         initial_results = relation.activate { |rel| klass.connection.select_all(rel) }
         if initial_results.is_a?(Array)
           initial_results.each do |r|
-            r['average'] = type_cast_calculated_value_switchman(r['average'], column_name, 'average')
-            r['count'] = type_cast_calculated_value_switchman(r['count'], column_name, 'count')
+            r["average"] = type_cast_calculated_value_switchman(r["average"], column_name, "average")
+            r["count"] = type_cast_calculated_value_switchman(r["count"], column_name, "count")
           end
-          result = initial_results.map { |r| r['average'] * r['count'] }.sum / initial_results.map do |r|
-                                                                                 r['count']
-                                                                               end.sum
+          result = initial_results.sum { |r| r["average"] * r["count"] } / initial_results.sum do |r|
+                                                                             r["count"]
+                                                                           end
         else
-          result = type_cast_calculated_value_switchman(initial_results.first['average'], column_name, 'average')
+          result = type_cast_calculated_value_switchman(initial_results.first["average"], column_name, "average")
         end
         result
       end
@@ -90,7 +90,7 @@ module Switchman
             row[opts[:aggregate_alias]] = type_cast_calculated_value_switchman(
               row[opts[:aggregate_alias]], column_name, opts[:operation]
             )
-            row['count'] = row['count'].to_i if opts[:operation] == 'average'
+            row["count"] = row["count"].to_i if opts[:operation] == "average"
 
             opts[:group_columns].each do |aliaz, _type, group_column_name|
               if opts[:associated] && (aliaz == opts[:group_aliases].first)
@@ -106,10 +106,49 @@ module Switchman
         compact_grouped_calculation_rows(rows, opts)
       end
 
+      if ::Rails.version >= "7.1"
+        def ids
+          return super unless klass.sharded_primary_key?
+
+          if loaded?
+            result = records.map do |record|
+              Shard.relative_id_for(record._read_attribute(primary_key),
+                                    record.shard,
+                                    Shard.current(klass.connection_class_for_self))
+            end
+            return @async ? Promise::Complete.new(result) : result
+          end
+
+          if has_include?(primary_key)
+            relation = apply_join_dependency.group(primary_key)
+            return relation.ids
+          end
+
+          columns = arel_columns([primary_key])
+          base_shard = Shard.current(klass.connection_class_for_self)
+          activate do |r|
+            relation = r.spawn
+            relation.select_values = columns
+
+            result = if relation.where_clause.contradiction?
+                       ::ActiveRecord::Result.empty
+                     else
+                       skip_query_cache_if_necessary do
+                         klass.connection.select_all(relation, "#{klass.name} Ids", async: @async)
+                       end
+                     end
+
+            result.then do |res|
+              type_cast_pluck_values(res, columns).map { |id| Shard.relative_id_for(id, Shard.current, base_shard) }
+            end
+          end
+        end
+      end
+
       private
 
       def type_cast_calculated_value_switchman(value, column_name, operation)
-        if ::Rails.version < '7.0'
+        if ::Rails.version < "7.0"
           type_cast_calculated_value(value, operation) do |val|
             column = aggregate_column(column_name)
             type ||= column.try(:type_caster) ||
@@ -125,39 +164,55 @@ module Switchman
       end
 
       def column_name_for(field)
-        field.respond_to?(:name) ? field.name.to_s : field.to_s.split('.').last
+        field.respond_to?(:name) ? field.name.to_s : field.to_s.split(".").last
       end
 
       def grouped_calculation_options(operation, column_name, distinct)
         opts = { operation: operation, column_name: column_name, distinct: distinct }
 
-        opts[:aggregate_alias] = aggregate_alias_for(operation, column_name)
+        # Rails 7.0.5
+        if defined?(::ActiveRecord::Calculations::ColumnAliasTracker)
+          column_alias_tracker = ::ActiveRecord::Calculations::ColumnAliasTracker.new(connection)
+        end
+
+        opts[:aggregate_alias] = aggregate_alias_for(operation, column_name, column_alias_tracker)
         group_attrs = group_values
         if group_attrs.first.respond_to?(:to_sym)
           association  = klass.reflect_on_association(group_attrs.first.to_sym)
-          associated   = group_attrs.size == 1 && association && association.macro == :belongs_to # only count belongs_to associations
+          # only count belongs_to associations
+          associated   = group_attrs.size == 1 && association && association.macro == :belongs_to
           group_fields = Array(associated ? association.foreign_key : group_attrs)
         else
           group_fields = group_attrs
         end
 
-        # to_s is because Rails 5 returns a string but Rails 6 returns a symbol.
-        group_aliases = group_fields.map { |field| column_alias_for(field.downcase.to_s).to_s }
+        group_aliases = group_fields.map do |field|
+          field = connection.visitor.compile(field) if ::Arel.arel_node?(field)
+          if column_alias_tracker
+            column_alias_tracker.alias_for(field.to_s.downcase)
+          else
+            column_alias_for(field.to_s.downcase)
+          end
+        end
         group_columns = group_aliases.zip(group_fields).map do |aliaz, field|
           [aliaz, type_for(field), column_name_for(field)]
         end
-        opts.merge!(association: association, associated: associated,
-                    group_aliases: group_aliases, group_columns: group_columns,
+        opts.merge!(association: association,
+                    associated: associated,
+                    group_aliases: group_aliases,
+                    group_columns: group_columns,
                     group_fields: group_fields)
 
         opts
       end
 
-      def aggregate_alias_for(operation, column_name)
-        if operation == 'count' && column_name == :all
-          'count_all'
-        elsif operation == 'average'
-          'average'
+      def aggregate_alias_for(operation, column_name, column_alias_tracker)
+        if operation == "count" && column_name == :all
+          "count_all"
+        elsif operation == "average"
+          "average"
+        elsif column_alias_tracker
+          column_alias_tracker.alias_for("#{operation} #{column_name}")
         else
           column_alias_for("#{operation} #{column_name}")
         end
@@ -173,13 +228,14 @@ module Switchman
             opts[:distinct]
           ).as(opts[:aggregate_alias])
         ]
-        if opts[:operation] == 'average'
+        if opts[:operation] == "average"
           # include count in average so we can recalculate the average
           # across all shards if needed
           select_values << operation_over_aggregate_column(
             aggregate_column(opts[:column_name]),
-            'count', opts[:distinct]
-          ).as('count')
+            "count",
+            opts[:distinct]
+          ).as("count")
         end
 
         haves = having_clause.send(:predicates)
@@ -205,22 +261,22 @@ module Switchman
           key = key.first if key.size == 1
           value = row[opts[:aggregate_alias]]
 
-          if opts[:operation] == 'average'
+          if opts[:operation] == "average"
             if result.key?(key)
               old_value, old_count = result[key]
-              new_count = old_count + row['count']
-              new_value = ((old_value * old_count) + (value * row['count'])) / new_count
+              new_count = old_count + row["count"]
+              new_value = ((old_value * old_count) + (value * row["count"])) / new_count
               result[key] = [new_value, new_count]
             else
-              result[key] = [value, row['count']]
+              result[key] = [value, row["count"]]
             end
           elsif result.key?(key)
             case opts[:operation]
-            when 'count', 'sum'
+            when "count", "sum"
               result[key] += value
-            when 'minimum'
+            when "minimum"
               result[key] = value if value < result[key]
-            when 'maximum'
+            when "maximum"
               result[key] = value if value > result[key]
             end
           else
@@ -228,7 +284,7 @@ module Switchman
           end
         end
 
-        result.transform_values!(&:first) if opts[:operation] == 'average'
+        result.transform_values!(&:first) if opts[:operation] == "average"
 
         result
       end

data/lib/switchman/active_record/connection_handler.rb

@@ -0,0 +1,18 @@
+# frozen_string_literal: true
+
+module Switchman
+  module ActiveRecord
+    module ConnectionHandler
+      def resolve_pool_config(config, connection_name, role, shard)
+        ret = super
+        # Make *all* pool configs use the same schema reflection
+        ret.schema_reflection = ConnectionHandler.global_schema_reflection
+        ret
+      end
+
+      def self.global_schema_reflection
+        @global_schema_reflection ||= ::ActiveRecord::ConnectionAdapters::SchemaReflection.new(nil)
+      end
+    end
+  end
+end

data/lib/switchman/active_record/connection_pool.rb

@@ -3,22 +3,24 @@
 module Switchman
   module ActiveRecord
     module ConnectionPool
-      def get_schema_cache(connection)
-        self.schema_cache ||= SharedSchemaCache.get_schema_cache(connection)
-        self.schema_cache.connection = connection
+      if ::Rails.version < "7.1"
+        def get_schema_cache(connection)
+          self.schema_cache ||= SharedSchemaCache.get_schema_cache(connection)
+          self.schema_cache.connection = connection
 
-        self.schema_cache
-      end
+          self.schema_cache
+        end
 
-      # rubocop:disable Naming/AccessorMethodName override method
-      def set_schema_cache(cache)
-        schema_cache = get_schema_cache(cache.connection)
+        # rubocop:disable Naming/AccessorMethodName override method
+        def set_schema_cache(cache)
+          schema_cache = get_schema_cache(cache.connection)
 
-        cache.instance_variables.each do |x|
-          schema_cache.instance_variable_set(x, cache.instance_variable_get(x))
+          cache.instance_variables.each do |x|
+            schema_cache.instance_variable_set(x, cache.instance_variable_get(x))
+          end
         end
+        # rubocop:enable Naming/AccessorMethodName override method
       end
-      # rubocop:enable Naming/AccessorMethodName override method
 
       def default_schema
         connection unless @schemas
@@ -48,7 +50,9 @@ module Switchman
       end
 
       def switch_database(conn)
-        @schemas = conn.current_schemas if !@schemas && conn.adapter_name == 'PostgreSQL' && !current_shard.database_server.config[:shard_name]
+        if !@schemas && conn.adapter_name == "PostgreSQL" && !current_shard.database_server.config[:shard_name]
+          @schemas = conn.current_schemas
+        end
 
         conn.shard = current_shard
       end
@@ -56,11 +60,11 @@ module Switchman
       private
 
       def current_shard
-        ::Rails.version < '7.0' ? connection_klass.current_switchman_shard : connection_class.current_switchman_shard
+        (::Rails.version < "7.0") ? connection_klass.current_switchman_shard : connection_class.current_switchman_shard
       end
 
       def tls_key
-        "#{object_id}_shard".to_sym
+        :"#{object_id}_shard"
       end
     end
   end

data/lib/switchman/active_record/database_configurations.rb

@@ -7,14 +7,26 @@ module Switchman
       # since all should point to the same data, even if multiple are writable
       # (Picks 'primary' since it is guaranteed to exist and switchman handles activating
       # deploy through other means)
-      def configs_for(include_replicas: false, name: nil, **)
-        res = super
-        if name && !include_replicas
-          return nil unless name.end_with?('primary')
-        elsif !include_replicas
-          return res.select { |config| config.name.end_with?('primary') }
+      if ::Rails.version < "7.1"
+        def configs_for(include_replicas: false, name: nil, **)
+          res = super
+          if name && !include_replicas
+            return nil unless name.end_with?("primary")
+          elsif !include_replicas
+            return res.select { |config| config.name.end_with?("primary") }
+          end
+          res
+        end
+      else
+        def configs_for(include_hidden: false, name: nil, **)
+          res = super
+          if name && !include_hidden
+            return nil unless name.end_with?("primary")
+          elsif !include_hidden
+            return res.select { |config| config.name.end_with?("primary") }
+          end
+          res
         end
-        res
       end
 
       private
@@ -27,21 +39,31 @@ module Switchman
         return configs if configs.is_a?(Array)
 
         db_configs = configs.flat_map do |env_name, config|
-          # It would be nice to do the auto-fallback that we want here, but we haven't
-          # actually done that for years (or maybe ever) and it will be a big lift to get working
-          roles = config.keys.select { |k| config[k].is_a?(Hash) || (config[k].is_a?(Array) && config[k].all? { |ck| ck.is_a?(Hash) }) }
-          base_config = config.except(*roles)
+          if config.is_a?(Hash)
+            # It would be nice to do the auto-fallback that we want here, but we haven't
+            # actually done that for years (or maybe ever) and it will be a big lift to get working
+            roles = config.keys.select do |k|
+              config[k].is_a?(Hash) || (config[k].is_a?(Array) && config[k].all?(Hash))
+            end
+            base_config = config.except(*roles)
+          else
+            base_config = config
+            roles = []
+          end
 
           name = "#{env_name}/primary"
-          name = 'primary' if env_name == default_env
+          name = "primary" if env_name == default_env
           base_db = build_db_config_from_raw_config(env_name, name, base_config)
           [base_db] + roles.map do |role|
-            build_db_config_from_raw_config(env_name, "#{env_name}/#{role}",
-                                            base_config.merge(config[role].is_a?(Array) ? config[role].first : config[role]))
+            build_db_config_from_raw_config(
+              env_name,
+              "#{env_name}/#{role}",
+              base_config.merge(config[role].is_a?(Array) ? config[role].first : config[role])
+            )
           end
         end
 
-        db_configs << environment_url_config(default_env, 'primary', {}) unless db_configs.find(&:for_current_env?)
+        db_configs << environment_url_config(default_env, "primary", {}) unless db_configs.find(&:for_current_env?)
 
         merge_db_environment_variables(default_env, db_configs.compact)
       end