switchman 2.1.6 → 2.2.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 32a8870f7b32fb16b98edcd9cd0930c30359df0a71a73ecc63752b256b5ac616
-  data.tar.gz: 90a3f31b1b8c5951720c7a45a3418c8448a5e08160939de73d487bfd8c9a7dc3
+  metadata.gz: c6658f02bb242697d5c6b9d7e9c57ba3613295fddc1de3d618cb74ed125835aa
+  data.tar.gz: 05fa7e267a2d442d3f9862e041fe6f297e3272693dd15273bfb17ec86187450f
 SHA512:
-  metadata.gz: 017200b98b0c53d0004c2632322e63141284716609b50400bf5936bf776aa06a68a648e1ff11df0d2d715e1993e051ea0e103c0356d0658ff121283447a376e7
-  data.tar.gz: 50cb3e155dd07d2d5e9171f788abce42b29b39d1f488d75cb97ae75842eda94876c4a9b0af134f1b0e03094bc14358c4b9ccc0677a83ebff741d948b9b496d5e
+  metadata.gz: 3651b8ad5d882571716507a49e264b90d70a318e436e816e9dbe2ec0b9ff70031f4bfa7fce4e04748bad8e4dad88a36d16eac4afa1b0c61e2f686ed138bb2ef5
+  data.tar.gz: d1d98111fc70a2786768b61856ae4a54ba914275628489d99f12b7de7b0a891d62e7c41189aad2a388941a0c9ce89742d6751c4078ff15b57660d63d390978ce

data/app/models/switchman/shard.rb

@@ -144,10 +144,9 @@ module Switchman
       # * +categories+ - an array of categories to activate
       # * +options+ -
       #    :parallel - true/false to execute in parallel, or a integer of how many
-      #                sub-processes per database server. Note that parallel
-      #                invocation currently uses forking, so should be used sparingly
-      #                because errors are not raised, and you cannot get results back
-      #    :max_procs - only run this many parallel processes at a time
+      #                sub-processes. Note that parallel invocation currently uses
+      #                forking, so should be used sparingly because you cannot get
+      #                results back
       #    :exception - :ignore, :raise, :defer (wait until the end and raise the first
       #                error), or a proc
       def with_each_shard(*args)
@@ -168,11 +167,14 @@ module Switchman
           scope, categories = args
         end
 
+        # back-compat
+        options[:parallel] = options.delete(:max_procs) if options.key?(:max_procs)
+
         parallel = case options[:parallel]
                      when true
-                       1
+                       [Environment.cpu_count || 2, 2].min
                      when false, nil
-                       0
+                       1
                      else
                        options[:parallel]
                    end
@@ -183,47 +185,19 @@ module Switchman
           scope = scope.order(::Arel.sql("database_server_id IS NOT NULL, database_server_id, id"))
         end
 
-        if parallel > 0
-          max_procs = determine_max_procs(options.delete(:max_procs), parallel)
+        if parallel > 1
           if ::ActiveRecord::Relation === scope
             # still need a post-uniq, cause the default database server could be NULL or Rails.env in the db
             database_servers = scope.reorder('database_server_id').select(:database_server_id).distinct.
                 map(&:database_server).compact.uniq
             # nothing to do
             return if database_servers.count == 0
-            parallel = [(max_procs.to_f / database_servers.count).ceil, parallel].min if max_procs
 
             scopes = Hash[database_servers.map do |server|
-              server_scope = server.shards.merge(scope)
-              if parallel == 1
-                subscopes = [server_scope]
-              else
-                subscopes = []
-                total = server_scope.count
-                ranges = []
-                server_scope.find_ids_in_ranges(:batch_size => (total.to_f / parallel).ceil) do |min, max|
-                  ranges << [min, max]
-                end
-                # create a half-open range on the last one
-                ranges.last[1] = nil
-                ranges.each do |min, max|
-                  subscope = server_scope.where("id>=?", min)
-                  subscope = subscope.where("id<=?", max) if max
-                  subscopes << subscope
-                end
-              end
-              [server, subscopes]
+              [server, server.shards.merge(scope)]
             end]
           else
             scopes = scope.group_by(&:database_server)
-            if parallel > 1
-              parallel = [(max_procs.to_f / scopes.count).ceil, parallel].min if max_procs
-              scopes = Hash[scopes.map do |(server, shards)|
-                [server, shards.in_groups(parallel, false).compact]
-              end]
-            else
-              scopes = Hash[scopes.map { |(server, shards)| [server, [shards]] }]
-            end
           end
 
           exception_pipes = []
@@ -249,8 +223,8 @@ module Switchman
           end
 
           # only one process; don't bother forking
-          if scopes.length == 1 && parallel == 1
-            return with_each_shard(scopes.first.last.first, categories, options) { yield }
+          if scopes.length == 1
+            return with_each_shard(scopes.first.last, categories, options) { yield }
           end
 
           # clear connections prior to forking (no more queries will be executed in the parent,
@@ -258,84 +232,78 @@ module Switchman
           # silly like dealloc'ing prepared statements)
           ::ActiveRecord::Base.clear_all_connections!
 
-          scopes.each do |server, subscopes|
-            subscopes.each_with_index do |subscope, idx|
-              if subscopes.length > 1
-                name = "#{server.id} #{idx + 1}"
-              else
-                name = server.id
-              end
+          scopes.each do |server, subscope|
+            name = server.id
+
+            exception_pipe = IO.pipe
+            exception_pipes << exception_pipe
+            pid, io_in, io_out, io_err = Open4.pfork4(lambda do
+              begin
+                Switchman.config[:on_fork_proc]&.call
+
+                # set a pretty name for the process title, up to 128 characters
+                # (we don't actually know the limit, depending on how the process
+                # was started)
+                # first, simplify the binary name by stripping directories,
+                # then truncate arguments as necessary
+                bin = File.basename($0)  # Process.argv0 doesn't work on Ruby 2.5 (https://bugs.ruby-lang.org/issues/15887)
+                max_length = 128 - bin.length - name.length - 3
+                args = ARGV.join(" ")
+                if max_length >= 0
+                  args = args[0..max_length]
+                end
+                new_title = [bin, args, name].join(" ")
+                Process.setproctitle(new_title)
 
-              exception_pipe = IO.pipe
-              exception_pipes << exception_pipe
-              pid, io_in, io_out, io_err = Open4.pfork4(lambda do
+                with_each_shard(subscope, categories, options) { yield }
+                exception_pipe.last.close
+              rescue Exception => e
                 begin
-                  Switchman.config[:on_fork_proc]&.call
-
-                  # set a pretty name for the process title, up to 128 characters
-                  # (we don't actually know the limit, depending on how the process
-                  # was started)
-                  # first, simplify the binary name by stripping directories,
-                  # then truncate arguments as necessary
-                  bin = File.basename($0)  # Process.argv0 doesn't work on Ruby 2.5 (https://bugs.ruby-lang.org/issues/15887)
-                  max_length = 128 - bin.length - name.length - 3
-                  args = ARGV.join(" ")
-                  if max_length >= 0
-                    args = args[0..max_length]
-                  end
-                  new_title = [bin, args, name].join(" ")
-                  Process.setproctitle(new_title)
-
-                  with_each_shard(subscope, categories, options) { yield }
-                  exception_pipe.last.close
-                rescue Exception => e
-                  begin
-                    dumped = Marshal.dump(e)
-                    dumped = nil if dumped.length > 64 * 1024
-                  rescue
-                    dumped = nil
-                  end
+                  dumped = Marshal.dump(e)
+                  dumped = nil if dumped.length > 64 * 1024
+                rescue
+                  dumped = nil
+                end
 
-                  if dumped.nil?
-                    # couldn't dump the exception; create a copy with just
-                    # the message and the backtrace
-                    e2 = e.class.new(e.message)
-                    backtrace = e.backtrace
-                    # truncate excessively long backtraces
-                    if backtrace.length > 50
-                      backtrace = backtrace[0...25] + ['...'] + backtrace[-25..-1]
-                    end
-                    e2.set_backtrace(backtrace)
-                    e2.instance_variable_set(:@active_shards, e.instance_variable_get(:@active_shards))
-                    dumped = Marshal.dump(e2)
+                if dumped.nil?
+                  # couldn't dump the exception; create a copy with just
+                  # the message and the backtrace
+                  e2 = e.class.new(e.message)
+                  backtrace = e.backtrace
+                  # truncate excessively long backtraces
+                  if backtrace.length > 50
+                    backtrace = backtrace[0...25] + ['...'] + backtrace[-25..-1]
                   end
-
-                  exception_pipe.last.set_encoding(dumped.encoding)
-                  exception_pipe.last.write(dumped)
-                  exception_pipe.last.flush
-                  exception_pipe.last.close
-                  exit! 1
+                  e2.set_backtrace(backtrace)
+                  e2.instance_variable_set(:@active_shards, e.instance_variable_get(:@active_shards))
+                  dumped = Marshal.dump(e2)
                 end
-              end)
-              exception_pipe.last.close
-              pids << pid
-              io_in.close # don't care about writing to stdin
-              out_fds << io_out
-              err_fds << io_err
-              pid_to_name_map[pid] = name
-              fd_to_name_map[io_out] = name
-              fd_to_name_map[io_err] = name
-
-              while max_procs && pids.count >= max_procs
-                while max_procs && out_fds.count >= max_procs
-                  # wait for output if we've hit the max_procs limit
-                  wait_for_output.call(out_fds, err_fds, fd_to_name_map)
-                end
-                # we've gotten all the output from one fd so wait for its child process to exit
-                found_pid, status = Process.wait2
-                pids.delete(found_pid)
-                errors << pid_to_name_map[found_pid] if status.exitstatus != 0
+
+                exception_pipe.last.set_encoding(dumped.encoding)
+                exception_pipe.last.write(dumped)
+                exception_pipe.last.flush
+                exception_pipe.last.close
+                exit! 1
+              end
+            end)
+            exception_pipe.last.close
+            pids << pid
+            io_in.close # don't care about writing to stdin
+            out_fds << io_out
+            err_fds << io_err
+            pid_to_name_map[pid] = name
+            fd_to_name_map[io_out] = name
+            fd_to_name_map[io_err] = name
+
+            while pids.count >= parallel
+              while out_fds.count >= parallel
+                # wait for output if we've hit the parallel limit
+                wait_for_output.call(out_fds, err_fds, fd_to_name_map)
               end
+              # we've gotten all the output from one fd so wait for its child process to exit
+              found_pid, status = Process.wait2
+              pids.delete(found_pid)
+              errors << pid_to_name_map[found_pid] if status.exitstatus != 0
             end
           end
 
@@ -549,24 +517,6 @@ module Switchman
         shard || source_shard || Shard.current
       end
 
-      # given the provided option, determines whether we need to (and whether
-      # it's possible) to determine a reasonable default.
-      def determine_max_procs(max_procs_input, parallel_input=2)
-        max_procs = nil
-        if max_procs_input
-          max_procs = max_procs_input.to_i
-          max_procs = nil if max_procs == 0
-        else
-          return 1 if parallel_input.nil? || parallel_input < 1
-          cpus = Environment.cpu_count
-          if cpus && cpus > 0
-            max_procs = cpus * parallel_input
-          end
-        end
-
-        return max_procs
-      end
-
       private
       # in-process caching
       def cached_shards

data/lib/switchman/active_record/attribute_methods.rb

@@ -90,7 +90,7 @@ module Switchman
         def define_method_original_attribute(attr_name)
           if sharded_column?(attr_name)
             reflection = reflection_for_integer_attribute(attr_name)
-            if attr_name == "id" && ::Rails.version >= '5.1.2'
+            if attr_name == "id"
               return if self.method_defined?(:original_id)
               owner = self
             else

data/lib/switchman/active_record/connection_pool.rb

@@ -75,7 +75,9 @@ module Switchman
           end
           @available.clear
           @connections.each do |conn|
-            @available.add conn
+            # we do not want to be introducing leased
+            # connections into the available queue
+            @available.add(conn) unless conn.in_use?
           end
         end
       end

data/lib/switchman/active_record/log_subscriber.rb

@@ -20,10 +20,7 @@ module Switchman
         shard = "  [#{shard[:database_server_id]}:#{shard[:id]} #{shard[:env]}]" if shard
 
         unless (payload[:binds] || []).empty?
-          use_old_format = (::Rails.version < '5.1.5')
-          args = use_old_format ?
-            [payload[:binds], payload[:type_casted_binds]] :
-            [payload[:type_casted_binds]]
+          args = [payload[:type_casted_binds]]
           casted_params = type_casted_binds(*args)
           binds = "  " + payload[:binds].zip(casted_params).map { |attr, value|
             render_bind(attr, value)

data/lib/switchman/active_record/migration.rb

@@ -30,10 +30,13 @@ module Switchman
     end
 
     module Migrator
-      # significant change: hash shard id, not database name
+      # significant change: just return MIGRATOR_SALT directly
+      # especially if you're going through pgbouncer, the database
+      # name you're accessing may not be consistent. it is NOT allowed
+      # to run migrations against multiple shards in the same database
+      # concurrently
       def generate_migrator_advisory_lock_id
-        shard_name_hash = Zlib.crc32(Shard.current.name)
-        ::ActiveRecord::Migrator::MIGRATOR_SALT * shard_name_hash
+        ::ActiveRecord::Migrator::MIGRATOR_SALT
       end
 
       if ::Rails.version >= '6.0'

data/lib/switchman/active_record/query_cache.rb

@@ -19,7 +19,7 @@ module Switchman
                   connection_id: object_id,
                   cached: true
                 }
-                args[:type_casted_binds] = -> { type_casted_binds(binds) } if ::Rails.version >= '5.1.5'
+                args[:type_casted_binds] = -> { type_casted_binds(binds) }
                 ::ActiveSupport::Notifications.instrument(
                     "sql.active_record",
                     args

data/lib/switchman/active_record/query_methods.rb

@@ -78,6 +78,10 @@ module Switchman
         end
       end
 
+      def or(other)
+        super(other.shard(self.primary_shard))
+      end
+
       private
 
       if ::Rails.version >= '5.2'
@@ -258,6 +262,25 @@ module Switchman
             end)
           end
 
+          if predicate.is_a?(::Arel::Nodes::Grouping)
+            next predicate unless predicate.expr.is_a?(::Arel::Nodes::Or)
+
+            or_expr = predicate.expr
+            left_node = or_expr.left
+            right_node = or_expr.right
+            new_left_predicates, binds = transpose_predicates([left_node], source_shard, target_shard,
+              remove_nonlocal_primary_keys,
+              binds: binds,
+              dup_binds_on_mutation: dup_binds_on_mutation)
+            new_right_predicates, binds = transpose_predicates([right_node], source_shard, target_shard,
+              remove_nonlocal_primary_keys,
+              binds: binds,
+              dup_binds_on_mutation: dup_binds_on_mutation)
+
+            next predicate if new_left_predicates[0] == left_node && new_right_predicates[0] == right_node
+            next ::Arel::Nodes::Grouping.new ::Arel::Nodes::Or.new(new_left_predicates[0], new_right_predicates[0])
+          end
+
           next predicate unless predicate.is_a?(::Arel::Nodes::Binary)
           next predicate unless predicate.left.is_a?(::Arel::Attributes::Attribute)
           relation, column = relation_and_column(predicate.left)

data/lib/switchman/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Switchman
-  VERSION = "2.1.6"
+  VERSION = "2.2.3"
 end

data/lib/tasks/switchman.rake

@@ -46,7 +46,8 @@ module Switchman
     end
 
     def self.options
-      { parallel: ENV['PARALLEL'].to_i, max_procs: ENV['MAX_PARALLEL_PROCS'] }
+      # we still pass through both of these options for back-compat purposes
+      { parallel: ENV['PARALLEL']&.to_i, max_procs: ENV['MAX_PARALLEL_PROCS']&.to_i }
     end
 
     # categories - an array or proc, to activate as the current shard during the
@@ -89,7 +90,7 @@ module Switchman
                 nil
               end
             rescue => e
-              puts "Exception from #{e.current_shard.id}: #{e.current_shard.description}" if options[:parallel] != 0
+              puts "Exception from #{e.current_shard.id}: #{e.current_shard.description}" if options[:parallel].to_i != 0
               raise
             end
           end

metadata

@@ -1,16 +1,16 @@
 --- !ruby/object:Gem::Specification
 name: switchman
 version: !ruby/object:Gem::Version
-  version: 2.1.6
+  version: 2.2.3
 platform: ruby
 authors:
 - Cody Cutrer
 - James Williams
 - Jacob Fugal
-autorequire:
+autorequire: 
 bindir: bin
 cert_chain: []
-date: 2021-08-11 00:00:00.000000000 Z
+date: 2022-03-09 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: railties
@@ -18,7 +18,7 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '5.1'
+        version: '5.2'
     - - "<"
       - !ruby/object:Gem::Version
         version: '6.1'
@@ -28,7 +28,7 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '5.1'
+        version: '5.2'
     - - "<"
       - !ruby/object:Gem::Version
         version: '6.1'
@@ -38,7 +38,7 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '5.1'
+        version: '5.2'
     - - "<"
       - !ruby/object:Gem::Version
         version: '6.1'
@@ -48,7 +48,7 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '5.1'
+        version: '5.2'
     - - "<"
       - !ruby/object:Gem::Version
         version: '6.1'
@@ -257,7 +257,7 @@ homepage: http://www.instructure.com/
 licenses:
 - MIT
 metadata: {}
-post_install_message:
+post_install_message: 
 rdoc_options: []
 require_paths:
 - lib
@@ -265,15 +265,15 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: '2.5'
+      version: '2.6'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.2.24
-signing_key:
+rubygems_version: 3.1.4
+signing_key: 
 specification_version: 4
 summary: Rails sharding magic
 test_files: []