switchman 2.1.6 → 2.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 32a8870f7b32fb16b98edcd9cd0930c30359df0a71a73ecc63752b256b5ac616
-  data.tar.gz: 90a3f31b1b8c5951720c7a45a3418c8448a5e08160939de73d487bfd8c9a7dc3
+  metadata.gz: cd39304f0dfae8329f95979c1cb2075d09a73cace1b2befa5f62c3100e863ca2
+  data.tar.gz: 742dd9cdbd63e3124b9ae4a9e0d73b56a071c43011f3ec0cdf43681f4977c116
 SHA512:
-  metadata.gz: 017200b98b0c53d0004c2632322e63141284716609b50400bf5936bf776aa06a68a648e1ff11df0d2d715e1993e051ea0e103c0356d0658ff121283447a376e7
-  data.tar.gz: 50cb3e155dd07d2d5e9171f788abce42b29b39d1f488d75cb97ae75842eda94876c4a9b0af134f1b0e03094bc14358c4b9ccc0677a83ebff741d948b9b496d5e
+  metadata.gz: 5355bcac300f60b1c2d626998ebd97a52871c4beff4bc597d2acc1f0439b7365b1a7478f91ec5a05bd8faa2df545e5444341ac296c826f1ab701372180a0282c
+  data.tar.gz: 54a69a012b6d48b91efd5417a71a536a9da8b02283284c5e9327121dcce0c458a1f065d6357012b9cbeba1dbba0603357a64505e69a16de886b9d34add5d8976

data/app/models/switchman/shard.rb

@@ -144,10 +144,9 @@ module Switchman
       # * +categories+ - an array of categories to activate
       # * +options+ -
       #    :parallel - true/false to execute in parallel, or a integer of how many
-      #                sub-processes per database server. Note that parallel
-      #                invocation currently uses forking, so should be used sparingly
-      #                because errors are not raised, and you cannot get results back
-      #    :max_procs - only run this many parallel processes at a time
+      #                sub-processes. Note that parallel invocation currently uses
+      #                forking, so should be used sparingly because you cannot get
+      #                results back
       #    :exception - :ignore, :raise, :defer (wait until the end and raise the first
       #                error), or a proc
       def with_each_shard(*args)
@@ -168,11 +167,14 @@ module Switchman
           scope, categories = args
         end
 
+        # back-compat
+        options[:parallel] = options.delete(:max_procs) if options.key?(:max_procs)
+
         parallel = case options[:parallel]
                      when true
-                       1
+                       [Environment.cpu_count || 2, 2].min
                      when false, nil
-                       0
+                       1
                      else
                        options[:parallel]
                    end
@@ -183,47 +185,19 @@ module Switchman
           scope = scope.order(::Arel.sql("database_server_id IS NOT NULL, database_server_id, id"))
         end
 
-        if parallel > 0
-          max_procs = determine_max_procs(options.delete(:max_procs), parallel)
+        if parallel > 1
           if ::ActiveRecord::Relation === scope
             # still need a post-uniq, cause the default database server could be NULL or Rails.env in the db
             database_servers = scope.reorder('database_server_id').select(:database_server_id).distinct.
                 map(&:database_server).compact.uniq
             # nothing to do
             return if database_servers.count == 0
-            parallel = [(max_procs.to_f / database_servers.count).ceil, parallel].min if max_procs
 
             scopes = Hash[database_servers.map do |server|
-              server_scope = server.shards.merge(scope)
-              if parallel == 1
-                subscopes = [server_scope]
-              else
-                subscopes = []
-                total = server_scope.count
-                ranges = []
-                server_scope.find_ids_in_ranges(:batch_size => (total.to_f / parallel).ceil) do |min, max|
-                  ranges << [min, max]
-                end
-                # create a half-open range on the last one
-                ranges.last[1] = nil
-                ranges.each do |min, max|
-                  subscope = server_scope.where("id>=?", min)
-                  subscope = subscope.where("id<=?", max) if max
-                  subscopes << subscope
-                end
-              end
-              [server, subscopes]
+              [server, server.shards.merge(scope)]
             end]
           else
             scopes = scope.group_by(&:database_server)
-            if parallel > 1
-              parallel = [(max_procs.to_f / scopes.count).ceil, parallel].min if max_procs
-              scopes = Hash[scopes.map do |(server, shards)|
-                [server, shards.in_groups(parallel, false).compact]
-              end]
-            else
-              scopes = Hash[scopes.map { |(server, shards)| [server, [shards]] }]
-            end
           end
 
           exception_pipes = []
@@ -249,8 +223,8 @@ module Switchman
           end
 
           # only one process; don't bother forking
-          if scopes.length == 1 && parallel == 1
-            return with_each_shard(scopes.first.last.first, categories, options) { yield }
+          if scopes.length == 1
+            return with_each_shard(scopes.first.last, categories, options) { yield }
           end
 
           # clear connections prior to forking (no more queries will be executed in the parent,
@@ -258,84 +232,78 @@ module Switchman
           # silly like dealloc'ing prepared statements)
           ::ActiveRecord::Base.clear_all_connections!
 
-          scopes.each do |server, subscopes|
-            subscopes.each_with_index do |subscope, idx|
-              if subscopes.length > 1
-                name = "#{server.id} #{idx + 1}"
-              else
-                name = server.id
-              end
+          scopes.each do |server, subscope|
+            name = server.id
+
+            exception_pipe = IO.pipe
+            exception_pipes << exception_pipe
+            pid, io_in, io_out, io_err = Open4.pfork4(lambda do
+              begin
+                Switchman.config[:on_fork_proc]&.call
+
+                # set a pretty name for the process title, up to 128 characters
+                # (we don't actually know the limit, depending on how the process
+                # was started)
+                # first, simplify the binary name by stripping directories,
+                # then truncate arguments as necessary
+                bin = File.basename($0)  # Process.argv0 doesn't work on Ruby 2.5 (https://bugs.ruby-lang.org/issues/15887)
+                max_length = 128 - bin.length - name.length - 3
+                args = ARGV.join(" ")
+                if max_length >= 0
+                  args = args[0..max_length]
+                end
+                new_title = [bin, args, name].join(" ")
+                Process.setproctitle(new_title)
 
-              exception_pipe = IO.pipe
-              exception_pipes << exception_pipe
-              pid, io_in, io_out, io_err = Open4.pfork4(lambda do
+                with_each_shard(subscope, categories, options) { yield }
+                exception_pipe.last.close
+              rescue Exception => e
                 begin
-                  Switchman.config[:on_fork_proc]&.call
-
-                  # set a pretty name for the process title, up to 128 characters
-                  # (we don't actually know the limit, depending on how the process
-                  # was started)
-                  # first, simplify the binary name by stripping directories,
-                  # then truncate arguments as necessary
-                  bin = File.basename($0)  # Process.argv0 doesn't work on Ruby 2.5 (https://bugs.ruby-lang.org/issues/15887)
-                  max_length = 128 - bin.length - name.length - 3
-                  args = ARGV.join(" ")
-                  if max_length >= 0
-                    args = args[0..max_length]
-                  end
-                  new_title = [bin, args, name].join(" ")
-                  Process.setproctitle(new_title)
-
-                  with_each_shard(subscope, categories, options) { yield }
-                  exception_pipe.last.close
-                rescue Exception => e
-                  begin
-                    dumped = Marshal.dump(e)
-                    dumped = nil if dumped.length > 64 * 1024
-                  rescue
-                    dumped = nil
-                  end
+                  dumped = Marshal.dump(e)
+                  dumped = nil if dumped.length > 64 * 1024
+                rescue
+                  dumped = nil
+                end
 
-                  if dumped.nil?
-                    # couldn't dump the exception; create a copy with just
-                    # the message and the backtrace
-                    e2 = e.class.new(e.message)
-                    backtrace = e.backtrace
-                    # truncate excessively long backtraces
-                    if backtrace.length > 50
-                      backtrace = backtrace[0...25] + ['...'] + backtrace[-25..-1]
-                    end
-                    e2.set_backtrace(backtrace)
-                    e2.instance_variable_set(:@active_shards, e.instance_variable_get(:@active_shards))
-                    dumped = Marshal.dump(e2)
+                if dumped.nil?
+                  # couldn't dump the exception; create a copy with just
+                  # the message and the backtrace
+                  e2 = e.class.new(e.message)
+                  backtrace = e.backtrace
+                  # truncate excessively long backtraces
+                  if backtrace.length > 50
+                    backtrace = backtrace[0...25] + ['...'] + backtrace[-25..-1]
                   end
-
-                  exception_pipe.last.set_encoding(dumped.encoding)
-                  exception_pipe.last.write(dumped)
-                  exception_pipe.last.flush
-                  exception_pipe.last.close
-                  exit! 1
+                  e2.set_backtrace(backtrace)
+                  e2.instance_variable_set(:@active_shards, e.instance_variable_get(:@active_shards))
+                  dumped = Marshal.dump(e2)
                 end
-              end)
-              exception_pipe.last.close
-              pids << pid
-              io_in.close # don't care about writing to stdin
-              out_fds << io_out
-              err_fds << io_err
-              pid_to_name_map[pid] = name
-              fd_to_name_map[io_out] = name
-              fd_to_name_map[io_err] = name
-
-              while max_procs && pids.count >= max_procs
-                while max_procs && out_fds.count >= max_procs
-                  # wait for output if we've hit the max_procs limit
-                  wait_for_output.call(out_fds, err_fds, fd_to_name_map)
-                end
-                # we've gotten all the output from one fd so wait for its child process to exit
-                found_pid, status = Process.wait2
-                pids.delete(found_pid)
-                errors << pid_to_name_map[found_pid] if status.exitstatus != 0
+
+                exception_pipe.last.set_encoding(dumped.encoding)
+                exception_pipe.last.write(dumped)
+                exception_pipe.last.flush
+                exception_pipe.last.close
+                exit! 1
+              end
+            end)
+            exception_pipe.last.close
+            pids << pid
+            io_in.close # don't care about writing to stdin
+            out_fds << io_out
+            err_fds << io_err
+            pid_to_name_map[pid] = name
+            fd_to_name_map[io_out] = name
+            fd_to_name_map[io_err] = name
+
+            while pids.count >= parallel
+              while out_fds.count >= parallel
+                # wait for output if we've hit the parallel limit
+                wait_for_output.call(out_fds, err_fds, fd_to_name_map)
               end
+              # we've gotten all the output from one fd so wait for its child process to exit
+              found_pid, status = Process.wait2
+              pids.delete(found_pid)
+              errors << pid_to_name_map[found_pid] if status.exitstatus != 0
             end
           end
 
@@ -549,24 +517,6 @@ module Switchman
         shard || source_shard || Shard.current
       end
 
-      # given the provided option, determines whether we need to (and whether
-      # it's possible) to determine a reasonable default.
-      def determine_max_procs(max_procs_input, parallel_input=2)
-        max_procs = nil
-        if max_procs_input
-          max_procs = max_procs_input.to_i
-          max_procs = nil if max_procs == 0
-        else
-          return 1 if parallel_input.nil? || parallel_input < 1
-          cpus = Environment.cpu_count
-          if cpus && cpus > 0
-            max_procs = cpus * parallel_input
-          end
-        end
-
-        return max_procs
-      end
-
       private
       # in-process caching
       def cached_shards

data/lib/switchman/active_record/migration.rb

@@ -30,10 +30,13 @@ module Switchman
     end
 
     module Migrator
-      # significant change: hash shard id, not database name
+      # significant change: just return MIGRATOR_SALT directly
+      # especially if you're going through pgbouncer, the database
+      # name you're accessing may not be consistent. it is NOT allowed
+      # to run migrations against multiple shards in the same database
+      # concurrently
       def generate_migrator_advisory_lock_id
-        shard_name_hash = Zlib.crc32(Shard.current.name)
-        ::ActiveRecord::Migrator::MIGRATOR_SALT * shard_name_hash
+        ::ActiveRecord::Migrator::MIGRATOR_SALT
       end
 
       if ::Rails.version >= '6.0'

data/lib/switchman/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Switchman
-  VERSION = "2.1.6"
+  VERSION = "2.2.0"
 end

data/lib/tasks/switchman.rake

@@ -46,6 +46,7 @@ module Switchman
     end
 
     def self.options
+      # we still pass through both of these options for back-compat purposes
       { parallel: ENV['PARALLEL'].to_i, max_procs: ENV['MAX_PARALLEL_PROCS'] }
     end
 

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: switchman
 version: !ruby/object:Gem::Version
-  version: 2.1.6
+  version: 2.2.0
 platform: ruby
 authors:
 - Cody Cutrer
@@ -10,7 +10,7 @@ authors:
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2021-08-11 00:00:00.000000000 Z
+date: 2021-08-20 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: railties