switchman 3.0.0 → 3.0.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: bb89193fead8142e30280a6ab2706f6e9af798ff18820fa144be3dd9ac77591f
-  data.tar.gz: e1157f5656f06ae9bdb7c62a2c9ae1611e2073cf47ca4278bae8605296603d21
+  metadata.gz: c3f9894955745b77bd418c8cede4882f5f37bf8cd3c0e2a3eecd53baa8658e1a
+  data.tar.gz: d160939ff0c15d4c57b0836abe0850e9c8a499fc05eb410932dc038d2835c60f
 SHA512:
-  metadata.gz: fbb9b4d9385dad87ba827400dc9245085c31e133d2445daf64b3d3e81f3102c189f8feafb328d085649146c8ccd2782fd7ba3cb410c262d06a31e014175cac99
-  data.tar.gz: 55c8010cbc73d112262d674d131b1fe0acbd39b5b482e73b9b7165aac23a25a93d5bbca4320a56b97450f68af8526045aede39c13cfb3426189073acba8f6a3f
+  metadata.gz: 89dc3d801b5f97d2147f10bcfb691279395590b45e9631103c73f66e487345e8608e185c07f5e54429bf43d29bd3f35c017bd93cd9354a88e29d0456d1d5040f
+  data.tar.gz: 9893296f2801ded4b1e227a4672b0e474735d2c6ba4ff7f5e95d0591b753659a97066f422ca0615d8abbd4c1e5f9bd4c8250ff9c3c697fcb18640b3ff47dd219

data/app/models/switchman/shard.rb

@@ -690,6 +690,7 @@ module Switchman
         Switchman.cache.delete(['shard', id].join('/'))
         Switchman.cache.delete('default_shard') if default?
       end
+      self.class.clear_cache
     end
 
     def default_name

data/db/migrate/20180828183945_add_default_shard_index.rb

@@ -10,6 +10,6 @@ class AddDefaultShardIndex < ActiveRecord::Migration[4.2]
               else
                 {}
               end
-    add_index :switchman_shards, :default, options
+    add_index :switchman_shards, :default, **options
   end
 end

data/lib/switchman.rb

@@ -17,4 +17,6 @@ module Switchman
   def self.cache=(cache)
     @cache = cache
   end
+
+  class OrderOnMultiShardQuery < RuntimeError; end
 end

data/lib/switchman/active_record/association.rb

@@ -85,7 +85,7 @@ module Switchman
         # Copypasta from Activerecord but with added global_id_for goodness.
         def records_for(ids)
           scope.where(association_key_name => ids).load do |record|
-            global_key = if record.class.connection_classes == UnshardedRecord
+            global_key = if model.connection_classes == UnshardedRecord
                            convert_key(record[association_key_name])
                          else
                            Shard.global_id_for(record[association_key_name], record.shard)

data/lib/switchman/active_record/attribute_methods.rb

@@ -141,6 +141,54 @@ module Switchman
         self.class.define_attribute_methods
         super
       end
+
+      # these are called if the specific methods haven't been defined yet
+      def attribute(attr_name)
+        return super unless self.class.sharded_column?(attr_name)
+
+        reflection = self.class.send(:reflection_for_integer_attribute, attr_name)
+        ::Switchman::Shard.relative_id_for(super, shard, ::Switchman::Shard.current(connection_classes_for_reflection(reflection)))
+      end
+
+      def attribute=(attr_name, new_value)
+        unless self.class.sharded_column?(attr_name)
+          super
+          return
+        end
+
+        reflection = self.class.send(:reflection_for_integer_attribute, attr_name)
+        super(::Switchman::Shard.relative_id_for(new_value, ::Switchman::Shard.current(connection_classes_for_reflection(reflection)), shard))
+      end
+
+      def global_attribute(attr_name)
+        if self.class.sharded_column?(attr_name)
+          ::Switchman::Shard.global_id_for(attribute(attr_name), shard)
+        else
+          attribute(attr_name)
+        end
+      end
+
+      def local_attribute(attr_name)
+        if self.class.sharded_column?(attr_name)
+          ::Switchman::Shard.local_id_for(attribute(attr_name), shard).first
+        else
+          attribute(attr_name)
+        end
+      end
+
+      private
+
+      def connection_classes_for_reflection(reflection)
+        if reflection
+          if reflection.options[:polymorphic]
+            read_attribute(reflection.foreign_type)&.constantize&.connection_classes
+          else
+            reflection.klass.connection_classes
+          end
+        else
+          self.class.connection_classes
+        end
+      end
     end
   end
 end

data/lib/switchman/active_record/calculations.rb

@@ -50,7 +50,7 @@ module Switchman
 
       def calculate_simple_average(column_name, distinct)
         # See activerecord#execute_simple_calculation
-        relation = reorder(nil)
+        relation = except(:order)
         column = aggregate_column(column_name)
         relation.select_values = [operation_over_aggregate_column(column, 'average', distinct).as('average'),
                                   operation_over_aggregate_column(column, 'count', distinct).as('count')]

data/lib/switchman/active_record/migration.rb

@@ -20,10 +20,22 @@ module Switchman
     end
 
     module Migrator
+      # significant change: hash shard name, not database name
       def generate_migrator_advisory_lock_id
-        shard_name_hash = Zlib.crc32("#{Shard.current.id}:#{Shard.current.name}")
+        shard_name_hash = Zlib.crc32(Shard.current.name)
         ::ActiveRecord::Migrator::MIGRATOR_SALT * shard_name_hash
       end
+
+      # significant change: strip out prefer_secondary from config
+      def with_advisory_lock_connection
+        pool = ::ActiveRecord::ConnectionAdapters::ConnectionHandler.new.establish_connection(
+          ::ActiveRecord::Base.connection_db_config.configuration_hash.except(:prefer_secondary)
+        )
+
+        pool.with_connection { |connection| yield(connection) } # rubocop:disable Style/ExplicitBlockArgument
+      ensure
+        pool&.disconnect!
+      end
     end
 
     module MigrationContext

data/lib/switchman/active_record/postgresql_adapter.rb

@@ -40,89 +40,36 @@ module Switchman
         select_values('SELECT * FROM unnest(current_schemas(false))')
       end
 
-      def tables(_name = nil)
-        query(<<-SQL, 'SCHEMA').map { |row| row[0] }
-          SELECT tablename
-          FROM pg_tables
-          WHERE schemaname = '#{shard.name}'
-        SQL
-      end
-
       def extract_schema_qualified_name(string)
         name = ::ActiveRecord::ConnectionAdapters::PostgreSQL::Utils.extract_schema_qualified_name(string.to_s)
         name.instance_variable_set(:@schema, shard.name) if string && !name.schema
         [name.schema, name.identifier]
       end
 
-      def view_exists?(name)
-        name = ::ActiveRecord::ConnectionAdapters::PostgreSQL::Utils.extract_schema_qualified_name(name.to_s)
-        return false unless name.identifier
-
-        name.instance_variable_set(:@schema, shard.name) unless name.schema
-
-        select_values(<<-SQL, 'SCHEMA').any?
-          SELECT c.relname
-          FROM pg_class c
-          LEFT JOIN pg_namespace n ON n.oid = c.relnamespace
-          WHERE c.relkind IN ('v','m') -- (v)iew, (m)aterialized view
-          AND c.relname = '#{name.identifier}'
-          AND n.nspname = '#{shard.name}'
-        SQL
-      end
-
-      def indexes(table_name)
-        result = query(<<-SQL, 'SCHEMA')
-           SELECT distinct i.relname, d.indisunique, d.indkey, pg_get_indexdef(d.indexrelid), t.oid
-           FROM pg_class t
-           INNER JOIN pg_index d ON t.oid = d.indrelid
-           INNER JOIN pg_class i ON d.indexrelid = i.oid
-           WHERE i.relkind = 'i'
-             AND d.indisprimary = 'f'
-             AND t.relname = '#{table_name}'
-             AND i.relnamespace IN (SELECT oid FROM pg_namespace WHERE nspname = '#{shard.name}' )
-          ORDER BY i.relname
-        SQL
-
-        result.map do |row|
-          index_name = row[0]
-          unique = row[1] == true || row[1] == 't'
-          indkey = row[2].split
-          inddef = row[3]
-          oid = row[4]
-
-          columns = Hash[query(<<-SQL, 'SCHEMA')] # rubocop:disable Style/HashConversion
-          SELECT a.attnum, a.attname
-          FROM pg_attribute a
-          WHERE a.attrelid = #{oid}
-          AND a.attnum IN (#{indkey.join(',')})
-          SQL
-
-          column_names = columns.stringify_keys.values_at(*indkey).compact
-
-          next if column_names.empty?
-
-          # add info on sort order for columns (only desc order is explicitly specified, asc is the default)
-          desc_order_columns = inddef.scan(/(\w+) DESC/).flatten
-          orders = desc_order_columns.any? ? Hash[desc_order_columns.map { |order_column| [order_column, :desc] }] : {} # rubocop:disable Style/HashConversion
-          where = inddef.scan(/WHERE (.+)$/).flatten[0]
-          using = inddef.scan(/USING (.+?) /).flatten[0].to_sym
-
-          ::ActiveRecord::ConnectionAdapters::IndexDefinition.new(table_name, index_name, unique, column_names,
-                                                                  orders: orders, where: where, using: using)
-        end.compact
+      # significant change: use the shard name if no explicit schema
+      def quoted_scope(name = nil, type: nil)
+        schema, name = extract_schema_qualified_name(name)
+        type = \
+          case type # rubocop:disable Style/HashLikeCase
+          when 'BASE TABLE'
+            "'r','p'"
+          when 'VIEW'
+            "'v','m'"
+          when 'FOREIGN TABLE'
+            "'f'"
+          end
+        scope = {}
+        scope[:schema] = quote(schema || shard.name)
+        scope[:name] = quote(name) if name
+        scope[:type] = type if type
+        scope
       end
 
-      def index_name_exists?(table_name, index_name, _default = nil)
-        exec_query(<<-SQL, 'SCHEMA').rows.first[0].to_i.positive?
-            SELECT COUNT(*)
-            FROM pg_class t
-            INNER JOIN pg_index d ON t.oid = d.indrelid
-            INNER JOIN pg_class i ON d.indexrelid = i.oid
-            WHERE i.relkind = 'i'
-              AND i.relname = '#{index_name}'
-              AND t.relname = '#{table_name}'
-              AND i.relnamespace IN (SELECT oid FROM pg_namespace WHERE nspname = '#{shard.name}' )
-        SQL
+      def foreign_keys(table_name)
+        super.each do |fk|
+          to_table_qualified_name = ::ActiveRecord::ConnectionAdapters::PostgreSQL::Utils.extract_schema_qualified_name(fk.to_table)
+          fk.to_table = to_table_qualified_name.identifier if to_table_qualified_name.schema == shard.name
+        end
       end
 
       def quote_local_table_name(name)
@@ -140,6 +87,10 @@ module Switchman
         name.quoted
       end
 
+      def with_global_table_name(&block)
+        with_local_table_name(false, &block)
+      end
+
       def with_local_table_name(enable = true) # rubocop:disable Style/OptionalBooleanParameter
         old_value = @use_local_table_name
         @use_local_table_name = enable
@@ -148,41 +99,6 @@ module Switchman
         @use_local_table_name = old_value
       end
 
-      def foreign_keys(table_name)
-        # mostly copy-pasted from AR - only change is to the nspname condition for qualified names support
-        fk_info = select_all <<-SQL.strip_heredoc
-          SELECT t2.oid::regclass::text AS to_table, a1.attname AS column, a2.attname AS primary_key, c.conname AS name, c.confupdtype AS on_update, c.confdeltype AS on_delete
-          FROM pg_constraint c
-          JOIN pg_class t1 ON c.conrelid = t1.oid
-          JOIN pg_class t2 ON c.confrelid = t2.oid
-          JOIN pg_attribute a1 ON a1.attnum = c.conkey[1] AND a1.attrelid = t1.oid
-          JOIN pg_attribute a2 ON a2.attnum = c.confkey[1] AND a2.attrelid = t2.oid
-          JOIN pg_namespace t3 ON c.connamespace = t3.oid
-          WHERE c.contype = 'f'
-            AND t1.relname = #{quote(table_name)}
-            AND t3.nspname = '#{shard.name}'
-          ORDER BY c.conname
-        SQL
-
-        fk_info.map do |row|
-          options = {
-            column: row['column'],
-            name: row['name'],
-            primary_key: row['primary_key']
-          }
-
-          options[:on_delete] = extract_foreign_key_action(row['on_delete'])
-          options[:on_update] = extract_foreign_key_action(row['on_update'])
-
-          # strip the schema name from to_table if it matches
-          to_table = row['to_table']
-          to_table_qualified_name = ::ActiveRecord::ConnectionAdapters::PostgreSQL::Utils.extract_schema_qualified_name(to_table)
-          to_table = to_table_qualified_name.identifier if to_table_qualified_name.schema == shard.name
-
-          ::ActiveRecord::ConnectionAdapters::ForeignKeyDefinition.new(table_name, to_table, options)
-        end
-      end
-
       def add_index_options(_table_name, _column_name, **)
         index, algorithm, if_not_exists = super
         algorithm = nil if DatabaseServer.creating_new_shard && algorithm == 'CONCURRENTLY'

data/lib/switchman/active_record/predicate_builder.rb

@@ -15,7 +15,7 @@ module Switchman
         def convert_to_id(value)
           case value
           when ::ActiveRecord::Base
-            value.send(primary_key)
+            value.id
           else
             super
           end

data/lib/switchman/active_record/query_methods.rb

@@ -239,6 +239,10 @@ module Switchman
         connection.with_local_table_name { super }
       end
 
+      def table_name_matches?(from)
+        connection.with_global_table_name { super }
+      end
+
       def transpose_predicates(predicates,
                                source_shard,
                                target_shard,
@@ -281,7 +285,7 @@ module Switchman
         remove = true if type == :primary &&
                          remove_nonlocal_primary_keys &&
                          predicate.left.relation.klass == klass &&
-                         predicate.is_a?(::Arel::Nodes::Equality)
+                         (predicate.is_a?(::Arel::Nodes::Equality) || predicate.is_a?(::Arel::Nodes::HomogeneousIn))
 
         current_source_shard =
           if source_shard
@@ -301,7 +305,7 @@ module Switchman
         new_right_value =
           case right
           when Array
-            right.map { |val| transpose_predicate_value(val, current_source_shard, target_shard, type, remove) }
+            right.map { |val| transpose_predicate_value(val, current_source_shard, target_shard, type, remove).presence }.compact
           else
             transpose_predicate_value(right, current_source_shard, target_shard, type, remove)
           end
@@ -315,7 +319,13 @@ module Switchman
             predicate.class.new(predicate.left, right.class.new(new_right_value, right.attribute))
           end
         elsif predicate.is_a?(::Arel::Nodes::HomogeneousIn)
-          predicate.class.new(new_right_value, predicate.attribute, predicate.type)
+          # switch to a regular In, so that Relation::WhereClause#contradiction? knows about it
+          if new_right_value.empty?
+            klass = predicate.type == :in ? ::Arel::Nodes::In : ::Arel::Nodes::NotIn
+            klass.new(predicate.attribute, new_right_value)
+          else
+            predicate.class.new(new_right_value, predicate.attribute, predicate.type)
+          end
         else
           predicate.class.new(predicate.left, new_right_value)
         end

data/lib/switchman/active_record/relation.rb

@@ -63,7 +63,7 @@ module Switchman
       %I[update_all delete_all].each do |method|
         class_eval <<-RUBY, __FILE__, __LINE__ + 1
           def #{method}(*args)
-            result = self.activate { |relation| relation.call_super(#{method.inspect}, Relation, *args) }
+            result = self.activate(unordered: true) { |relation| relation.call_super(#{method.inspect}, Relation, *args) }
             result = result.sum if result.is_a?(Array)
             result
           end
@@ -103,7 +103,7 @@ module Switchman
         end
       end
 
-      def activate(&block)
+      def activate(unordered: false, &block)
         shards = all_shards
         if Array === shards && shards.length == 1
           if shards.first == DefaultShard || shards.first == Shard.current(klass.connection_classes)
@@ -112,10 +112,61 @@ module Switchman
             shards.first.activate(klass.connection_classes) { yield(self, shards.first) }
           end
         else
-          # TODO: implement local limit to avoid querying extra shards
-          Shard.with_each_shard(shards, [klass.connection_classes]) do
-            shard(Shard.current(klass.connection_classes), :to_a).activate(&block)
+          result_count = 0
+          can_order = false
+          result = Shard.with_each_shard(shards, [klass.connection_classes]) do
+            # don't even query other shards if we're already past the limit
+            next if limit_value && result_count >= limit_value && order_values.empty?
+
+            relation = shard(Shard.current(klass.connection_classes), :to_a)
+            # do a minimal query if possible
+            relation = relation.limit(limit_value - result_count) if limit_value && !result_count.zero? && order_values.empty?
+
+            shard_results = relation.activate(&block)
+
+            if shard_results.present? && !unordered
+              can_order ||= can_order_cross_shard_results? unless order_values.empty?
+              raise OrderOnMultiShardQuery if !can_order && !order_values.empty? && result_count.positive?
+
+              result_count += shard_results.is_a?(Array) ? shard_results.length : 1
+            end
+            shard_results
+          end
+
+          result = reorder_cross_shard_results(result) if can_order
+          result.slice!(limit_value..-1) if limit_value
+          result
+        end
+      end
+
+      def can_order_cross_shard_results?
+        # we only presume to be able to post-sort the most basic of orderings
+        order_values.all? { |ov| ov.is_a?(::Arel::Nodes::Ordering) && ov.expr.is_a?(::Arel::Attributes::Attribute) }
+      end
+
+      def reorder_cross_shard_results(results)
+        results.sort! do |l, r|
+          result = 0
+          order_values.each do |ov|
+            if l.is_a?(::ActiveRecord::Base)
+              a = l.attribute(ov.expr.name)
+              b = r.attribute(ov.expr.name)
+            else
+              a, b = l, r
+            end
+            next if a == b
+
+            if a.nil? || b.nil?
+              result = 1 if a.nil?
+              result *= -1 if ov.is_a?(::Arel::Nodes::Descending)
+            else
+              result = a <=> b
+            end
+
+            result *= -1 if ov.is_a?(::Arel::Nodes::Descending)
+            break unless result.zero?
           end
+          result
         end
       end
     end

data/lib/switchman/database_server.rb

@@ -113,7 +113,7 @@ module Switchman
     end
 
     def config(environment = :primary)
-      @configs[environment] ||= begin
+      @configs[environment] ||=
         case @config[environment]
         when Array
           @config[environment].map do |config|
@@ -127,7 +127,6 @@ module Switchman
         else
           @config
         end
-      end
     end
 
     def guard_rail_environment
@@ -214,7 +213,7 @@ module Switchman
             ::ActiveRecord::Migration.verbose = false
 
             unless schema == false
-              shard.activate do
+              shard.activate(*Shard.sharded_models) do
                 reset_column_information
 
                 ::ActiveRecord::Base.connection.transaction(requires_new: true) do

data/lib/switchman/engine.rb

@@ -150,6 +150,7 @@ module Switchman
         ::ActiveRecord::Relation.include(CallSuper)
 
         ::ActiveRecord::PredicateBuilder::AssociationQueryValue.prepend(ActiveRecord::PredicateBuilder::AssociationQueryValue)
+        ::ActiveRecord::PredicateBuilder::PolymorphicArrayValue.prepend(ActiveRecord::PredicateBuilder::AssociationQueryValue)
 
         ::ActiveRecord::Tasks::DatabaseTasks.singleton_class.prepend(ActiveRecord::Tasks::DatabaseTasks)
 

data/lib/switchman/test_helper.rb

@@ -19,7 +19,7 @@ module Switchman
         end
 
         server1 = Shard.default.database_server
-        server2 = DatabaseServer.create(Shard.default.database_server.config)
+        server2 = DatabaseServer.create(Shard.default.database_server.config.merge(server2: true))
 
         if server1 == Shard.default.database_server && server1.config[:shard1] && server1.config[:shard2]
           # look for the shards in the db already

data/lib/switchman/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Switchman
-  VERSION = '3.0.0'
+  VERSION = '3.0.5'
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: switchman
 version: !ruby/object:Gem::Version
-  version: 3.0.0
+  version: 3.0.5
 platform: ruby
 authors:
 - Cody Cutrer
@@ -10,7 +10,7 @@ authors:
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2021-03-17 00:00:00.000000000 Z
+date: 2021-06-11 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activerecord
@@ -289,7 +289,6 @@ files:
 - lib/switchman/open4.rb
 - lib/switchman/r_spec_helper.rb
 - lib/switchman/rails.rb
-- lib/switchman/schema_cache.rb
 - lib/switchman/sharded_instrumenter.rb
 - lib/switchman/standard_error.rb
 - lib/switchman/test_helper.rb
@@ -314,7 +313,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.1.4
+rubygems_version: 3.2.15
 signing_key:
 specification_version: 4
 summary: Rails sharding magic

data/lib/switchman/schema_cache.rb

@@ -1,20 +0,0 @@
-# frozen_string_literal: true
-
-module Switchman
-  class SchemaCache < ::ActiveRecord::ConnectionAdapters::SchemaCache
-    delegate :connection, to: :pool
-    attr_reader :pool
-
-    def initialize(pool)
-      @pool = pool
-      super(nil)
-    end
-
-    def copy_values(other_cache)
-      # use the same cached values but still fall back to the correct pool
-      %i[@columns @columns_hash @primary_keys @data_sources].each do |iv|
-        instance_variable_set(iv, other_cache.instance_variable_get(iv))
-      end
-    end
-  end
-end