switchman 2.0.8 → 2.0.13

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 697ce16e90d84de2535fdbf6dacc40640323dddffa4becb4a3d5e8c3b4d1a67f
-  data.tar.gz: b94f2c678d8dd4d03c7dd089c441ced5a75923778c972a86a7cc3f20c126bc97
+  metadata.gz: de9d4ea9f0368bf1bfe15fdb8dd6bef040a8ec107aed386eaa81f8fa444513df
+  data.tar.gz: 71b7905b40d59a4767576552c8f5d16a2b3bdaec6d910f81f18d803a9c916256
 SHA512:
-  metadata.gz: 82dff35e613f3f9147e7575f1865402df57375dd8f8396ae6f61ad86cf82ec401098b4210eecabb8596c9d7dbba3a3a3ba87c030d7b17b74aae1605cfc3efac9
-  data.tar.gz: 8678d9c29a485baf8bc483d3c6717d4e2c88d2de14e51e4356a656b579bd0c65579274e0afd70a94d2fb1bc8c617b44a47d73bb04cd8a0b41f1ad2a378ff7e38
+  metadata.gz: bac07aa4f15521f123a1d95f676448d649c71eddd86fdfbbb57504f59d0c71f9ce69618aa7d1e25e93fdc278708cb2a7385d111417f23a8fc0eb1e9b6f6eefb3
+  data.tar.gz: 1c091615da88fe2f54de51de1e740336007fd13d66d0cac0b977d0e2770a6dfacd6a85bd12b689a0c610b1de5daeab4512743f0c50199e0f610806cc2f4dd629

data/app/models/switchman/shard.rb

@@ -718,6 +718,7 @@ module Switchman
         Switchman.cache.delete(['shard', id].join('/'))
         Switchman.cache.delete("default_shard") if default?
       end
+      self.class.clear_cache
     end
 
     def default_name

data/db/migrate/20180828183945_add_default_shard_index.rb

@@ -10,6 +10,6 @@ class AddDefaultShardIndex < ActiveRecord::Migration[4.2]
               else
                 {}
               end
-    add_index :switchman_shards, :default, options
+    add_index :switchman_shards, :default, **options
   end
 end

data/lib/switchman.rb

@@ -17,4 +17,6 @@ module Switchman
   def self.cache=(cache)
     @cache = cache
   end
+
+  class OrderOnMultiShardQuery < RuntimeError; end
 end

data/lib/switchman/active_record/association.rb

@@ -87,7 +87,7 @@ module Switchman
           # Copypasta from Activerecord but with added global_id_for goodness.
           def records_for(ids)
             scope.where(association_key_name => ids).load do |record|
-              global_key = if record.class.shard_category == :unsharded
+              global_key = if model.shard_category == :unsharded
                              convert_key(record[association_key_name])
                            else
                              Shard.global_id_for(record[association_key_name], record.shard)

data/lib/switchman/active_record/calculations.rb

@@ -51,7 +51,7 @@ module Switchman
 
       def calculate_simple_average(column_name, distinct)
         # See activerecord#execute_simple_calculation
-        relation = reorder(nil)
+        relation = except(:order)
         column = aggregate_column(column_name)
         relation.select_values = [operation_over_aggregate_column(column, "average", distinct).as("average"),
                                   operation_over_aggregate_column(column, "count", distinct).as("count")]

data/lib/switchman/active_record/connection_handler.rb

@@ -135,7 +135,7 @@ module Switchman
             primary_pool = retrieve_connection_pool("primary")
             if primary_pool.is_a?(ConnectionPoolProxy)
               pool = ConnectionPoolProxy.new(spec_name.to_sym, primary_pool.default_pool, @shard_connection_pools)
-              pool.schema_cache.copy_values(primary_pool.schema_cache)
+              pool.schema_cache.copy_references(primary_pool.schema_cache)
               pool
             else
               primary_pool

data/lib/switchman/active_record/migration.rb

@@ -30,10 +30,42 @@ module Switchman
     end
 
     module Migrator
+      # significant change: hash shard id, not database name
       def generate_migrator_advisory_lock_id
-        shard_name_hash = Zlib.crc32("#{Shard.current.id}:#{Shard.current.name}")
+        shard_name_hash = Zlib.crc32(Shard.current.name)
         ::ActiveRecord::Migrator::MIGRATOR_SALT * shard_name_hash
       end
+
+      if ::Rails.version >= '6.0'
+        # copy/paste from Rails 6.1
+        def with_advisory_lock
+          lock_id = generate_migrator_advisory_lock_id
+
+          with_advisory_lock_connection do |connection|
+            got_lock = connection.get_advisory_lock(lock_id)
+            raise ::ActiveRecord::ConcurrentMigrationError unless got_lock
+            load_migrated # reload schema_migrations to be sure it wasn't changed by another process before we got the lock
+            yield
+          ensure
+            if got_lock && !connection.release_advisory_lock(lock_id)
+              raise ::ActiveRecord::ConcurrentMigrationError.new(
+                ::ActiveRecord::ConcurrentMigrationError::RELEASE_LOCK_FAILED_MESSAGE
+              )
+            end
+          end
+        end
+
+        # significant change: strip out prefer_secondary from config
+        def with_advisory_lock_connection
+          pool = ::ActiveRecord::ConnectionAdapters::ConnectionHandler.new.establish_connection(
+            ::ActiveRecord::Base.connection_config.except(:prefer_secondary)
+          )
+
+          pool.with_connection { |connection| yield(connection) }
+        ensure
+          pool&.disconnect!
+        end
+      end
     end
 
     module MigrationContext

data/lib/switchman/active_record/postgresql_adapter.rb

@@ -215,6 +215,10 @@ module Switchman
         name.quoted
       end
 
+      def with_global_table_name(&block)
+        with_local_table_name(false, &block)
+      end
+
       def with_local_table_name(enable = true)
         old_value = @use_local_table_name
         @use_local_table_name = enable

data/lib/switchman/active_record/predicate_builder.rb

@@ -15,7 +15,7 @@ module Switchman
         def convert_to_id(value)
           case value
           when ::ActiveRecord::Base
-            value.send(primary_key)
+            value.id
           else
             super
           end

data/lib/switchman/active_record/query_methods.rb

@@ -233,6 +233,10 @@ module Switchman
         connection.with_local_table_name { super }
       end
 
+      def table_name_matches?(from)
+        connection.with_global_table_name { super }
+      end
+
       # semi-private
       public
       def transpose_predicates(predicates,
@@ -242,6 +246,18 @@ module Switchman
                                binds: nil,
                                dup_binds_on_mutation: false)
         result = predicates.map do |predicate|
+          if ::Rails.version >= '5.2' && predicate.is_a?(::Arel::Nodes::And)
+            new_predicates, _binds = transpose_predicates(predicate.children, source_shard, target_shard,
+              remove_nonlocal_primary_keys,
+              binds: binds,
+              dup_binds_on_mutation: dup_binds_on_mutation)
+            next (if new_predicates == predicate.children
+              predicate
+            else
+              ::Arel::Nodes::And.new(new_predicates)
+            end)
+          end
+
           next predicate unless predicate.is_a?(::Arel::Nodes::Binary)
           next predicate unless predicate.left.is_a?(::Arel::Attributes::Attribute)
           relation, column = relation_and_column(predicate.left)
@@ -250,7 +266,7 @@ module Switchman
           remove = true if type == :primary &&
               remove_nonlocal_primary_keys &&
               predicate.left.relation.model == klass &&
-              predicate.is_a?(::Arel::Nodes::Equality)
+              (predicate.is_a?(::Arel::Nodes::Equality) || predicate.is_a?(::Arel::Nodes::In))
 
           current_source_shard =
               if source_shard
@@ -265,7 +281,7 @@ module Switchman
             new_right_value =
               case predicate.right
               when Array
-                predicate.right.map {|val| transpose_predicate_value(val, current_source_shard, target_shard, type, remove) }
+                predicate.right.map {|val| transpose_predicate_value(val, current_source_shard, target_shard, type, remove).presence }.compact
               else
                 transpose_predicate_value(predicate.right, current_source_shard, target_shard, type, remove)
               end
@@ -340,7 +356,7 @@ module Switchman
             value
           else
             local_id = Shard.relative_id_for(current_id, current_shard, target_shard) || current_id
-            local_id = [] if remove_non_local_ids && local_id.is_a?(Integer) && local_id > Shard::IDS_PER_SHARD
+            return nil if remove_non_local_ids && local_id.is_a?(Integer) && local_id > Shard::IDS_PER_SHARD
             if current_id != local_id
               # make a new bind param
               ::Arel::Nodes::BindParam.new(query_att.class.new(query_att.name, local_id, query_att.type))

data/lib/switchman/active_record/relation.rb

@@ -62,7 +62,7 @@ module Switchman
       %I{update_all delete_all}.each do |method|
         class_eval <<-RUBY, __FILE__, __LINE__ + 1
           def #{method}(*args)
-            result = self.activate { |relation| relation.call_super(#{method.inspect}, Relation, *args) }
+            result = self.activate(unordered: true) { |relation| relation.call_super(#{method.inspect}, Relation, *args) }
             result = result.sum if result.is_a?(Array)
             result
           end
@@ -97,7 +97,7 @@ module Switchman
         end
       end
 
-      def activate(&block)
+      def activate(unordered: false, &block)
         shards = all_shards
         if (Array === shards && shards.length == 1)
           if shards.first == DefaultShard || shards.first == Shard.current(klass.shard_category)
@@ -106,10 +106,64 @@ module Switchman
             shards.first.activate(klass.shard_category) { yield(self, shards.first) }
           end
         else
-          # TODO: implement local limit to avoid querying extra shards
-          Shard.with_each_shard(shards, [klass.shard_category]) do
-            shard(Shard.current(klass.shard_category), :to_a).activate(&block)
+          result_count = 0
+          can_order = false
+          result = Shard.with_each_shard(shards, [klass.shard_category]) do
+            # don't even query other shards if we're already past the limit
+            next if limit_value && result_count >= limit_value && order_values.empty?
+
+            relation = shard(Shard.current(klass.shard_category), :to_a)
+            # do a minimal query if possible
+            relation = relation.limit(limit_value - result_count) if limit_value && !result_count.zero? && order_values.empty?
+
+            shard_results = relation.activate(&block)
+
+            if shard_results.present? && !unordered
+              can_order ||= can_order_cross_shard_results? unless order_values.empty?
+              raise OrderOnMultiShardQuery if !can_order && !order_values.empty? && result_count.positive?
+
+              result_count += shard_results.is_a?(Array) ? shard_results.length : 1
+            end
+            shard_results
+          end
+
+          result = reorder_cross_shard_results(result) if can_order
+          result.slice!(limit_value..-1) if limit_value
+          result
+        end
+      end
+
+      def can_order_cross_shard_results?
+        # we only presume to be able to post-sort the most basic of orderings
+        order_values.all? { |ov| ov.is_a?(::Arel::Nodes::Ordering) && ov.expr.is_a?(::Arel::Attributes::Attribute) }
+      end
+
+      def reorder_cross_shard_results(results)
+        results.sort! do |l, r|
+          result = 0
+          order_values.each do |ov|
+            if !l.is_a?(::ActiveRecord::Base)
+              a, b = l, r
+            elsif l.respond_to?(ov.expr.name)
+              a = l.send(ov.expr.name)
+              b = r.send(ov.expr.name)
+            else
+              a = l.attributes[ov.expr.name]
+              b = r.attributes[ov.expr.name]
+            end
+            next if a == b
+
+            if a.nil? || b.nil?
+              result = 1 if a.nil?
+              result *= -1 if ov.is_a?(::Arel::Nodes::Descending)
+            else
+              result = a <=> b
+            end
+
+            result *= -1 if ov.is_a?(::Arel::Nodes::Descending)
+            break unless result.zero?
           end
+          result
         end
       end
     end

data/lib/switchman/connection_pool_proxy.rb

@@ -84,6 +84,10 @@ module Switchman
       @schema_cache
     end
 
+    def set_schema_cache(cache)
+      @schema_cache.copy_values(cache)
+    end
+
     %w{release_connection
        disconnect!
        flush!

data/lib/switchman/engine.rb

@@ -142,6 +142,7 @@ module Switchman
 
         ::ActiveRecord::Relation::WhereClauseFactory.prepend(ActiveRecord::WhereClauseFactory)
         ::ActiveRecord::PredicateBuilder::AssociationQueryValue.prepend(ActiveRecord::PredicateBuilder::AssociationQueryValue)
+        ::ActiveRecord::PredicateBuilder::PolymorphicArrayValue.prepend(ActiveRecord::PredicateBuilder::AssociationQueryValue)
         ::ActiveRecord::TypeCaster::Map.include(ActiveRecord::TypeCaster::Map)
         ::ActiveRecord::TypeCaster::Connection.include(ActiveRecord::TypeCaster::Connection)
 

data/lib/switchman/schema_cache.rb

@@ -5,14 +5,22 @@ module Switchman
     delegate :connection, to: :pool
     attr_reader :pool
 
+    SHARED_IVS = %i{@columns @columns_hash @primary_keys @data_sources @indexes}.freeze
+
     def initialize(pool)
       @pool = pool
       super(nil)
     end
 
     def copy_values(other_cache)
+      SHARED_IVS.each do |iv|
+        instance_variable_get(iv).replace(other_cache.instance_variable_get(iv))
+      end
+    end
+
+    def copy_references(other_cache)
       # use the same cached values but still fall back to the correct pool
-      [:@columns, :@columns_hash, :@primary_keys, :@data_sources].each do |iv|
+      SHARED_IVS.each do |iv|
         instance_variable_set(iv, other_cache.instance_variable_get(iv))
       end
     end

data/lib/switchman/test_helper.rb

@@ -24,7 +24,7 @@ module Switchman
         else
           server1 = Shard.default.database_server
         end
-        server2 = DatabaseServer.create(Shard.default.database_server.config)
+        server2 = DatabaseServer.create(Shard.default.database_server.config.merge(server2: true))
 
         if server1 == Shard.default.database_server && server1.config[:shard1] && server1.config[:shard2]
           # look for the shards in the db already

data/lib/switchman/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Switchman
-  VERSION = "2.0.8"
+  VERSION = "2.0.13"
 end

metadata

@@ -1,16 +1,16 @@
 --- !ruby/object:Gem::Specification
 name: switchman
 version: !ruby/object:Gem::Version
-  version: 2.0.8
+  version: 2.0.13
 platform: ruby
 authors:
 - Cody Cutrer
 - James Williams
 - Jacob Fugal
-autorequire: 
+autorequire:
 bindir: bin
 cert_chain: []
-date: 2021-04-01 00:00:00.000000000 Z
+date: 2021-06-11 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: railties
@@ -257,7 +257,7 @@ homepage: http://www.instructure.com/
 licenses:
 - MIT
 metadata: {}
-post_install_message: 
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -272,8 +272,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.0.3
-signing_key: 
+rubygems_version: 3.2.15
+signing_key:
 specification_version: 4
 summary: Rails sharding magic
 test_files: []