switchman 2.1.5 → 2.2.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: e619ee38f22bd57ca2cd0c1fbb5ccc6f1a7f0257c71dcc4995d0032dc44865ad
-  data.tar.gz: 7f980b4cfd0cff41357b3bd0f0cd53eb8a41556d1b4bc31eb3e0d2316391bc55
+  metadata.gz: 901b8608f750cdaacfabbb90b2894acb1e73adeb27438ccff1907376f41bbadd
+  data.tar.gz: 5dc547e878ef50027f013370010372dcb90fdda3ada531b5d6dc85cb902db633
 SHA512:
-  metadata.gz: 679583144bfc68dc465f31eaedb32d8d960a7645cbf360c83562e463fc11174d3b7191c866246660340045a3fa0985086771e230d48dcbfd77a78b759225fa41
-  data.tar.gz: 9750de1c00f02b7307589128d70f614b9508588049bd9d384eb3f7e77d7171ac04a71ba2b0fd9700b1306ab68585e002807dcd183ba4b1e115d730f822dac0ce
+  metadata.gz: 5b16b9e6ea387ec65f117e29d9b12b02287b4403474342b593fcefed333cc816137a596af01fa63eb78e5f37a44381177071e95453c8ea2d0bbb0366a3ce8992
+  data.tar.gz: af01a3a853f9c54151da689ba408e8443ee40ba97402175ae0e0cdf961af6f6c9396c57dee668d2a2c885ad22d313fbe4de9804a063317628500ff1636487ef7

data/app/models/switchman/shard.rb

@@ -129,6 +129,11 @@ module Switchman
         cached_shards[id]
       end
 
+      def preload_cache
+        cached_shards.reverse_merge!(active_shards.values.index_by(&:id))
+        cached_shards.reverse_merge!(all.index_by(&:id))
+      end
+
       def clear_cache
         cached_shards.clear
       end
@@ -139,10 +144,9 @@ module Switchman
       # * +categories+ - an array of categories to activate
       # * +options+ -
       #    :parallel - true/false to execute in parallel, or a integer of how many
-      #                sub-processes per database server. Note that parallel
-      #                invocation currently uses forking, so should be used sparingly
-      #                because errors are not raised, and you cannot get results back
-      #    :max_procs - only run this many parallel processes at a time
+      #                sub-processes. Note that parallel invocation currently uses
+      #                forking, so should be used sparingly because you cannot get
+      #                results back
       #    :exception - :ignore, :raise, :defer (wait until the end and raise the first
       #                error), or a proc
       def with_each_shard(*args)
@@ -163,11 +167,14 @@ module Switchman
           scope, categories = args
         end
 
+        # back-compat
+        options[:parallel] = options.delete(:max_procs) if options.key?(:max_procs)
+
         parallel = case options[:parallel]
                      when true
-                       1
+                       [Environment.cpu_count || 2, 2].min
                      when false, nil
-                       0
+                       1
                      else
                        options[:parallel]
                    end
@@ -178,47 +185,19 @@ module Switchman
           scope = scope.order(::Arel.sql("database_server_id IS NOT NULL, database_server_id, id"))
         end
 
-        if parallel > 0
-          max_procs = determine_max_procs(options.delete(:max_procs), parallel)
+        if parallel > 1
           if ::ActiveRecord::Relation === scope
             # still need a post-uniq, cause the default database server could be NULL or Rails.env in the db
             database_servers = scope.reorder('database_server_id').select(:database_server_id).distinct.
                 map(&:database_server).compact.uniq
             # nothing to do
             return if database_servers.count == 0
-            parallel = [(max_procs.to_f / database_servers.count).ceil, parallel].min if max_procs
 
             scopes = Hash[database_servers.map do |server|
-              server_scope = server.shards.merge(scope)
-              if parallel == 1
-                subscopes = [server_scope]
-              else
-                subscopes = []
-                total = server_scope.count
-                ranges = []
-                server_scope.find_ids_in_ranges(:batch_size => (total.to_f / parallel).ceil) do |min, max|
-                  ranges << [min, max]
-                end
-                # create a half-open range on the last one
-                ranges.last[1] = nil
-                ranges.each do |min, max|
-                  subscope = server_scope.where("id>=?", min)
-                  subscope = subscope.where("id<=?", max) if max
-                  subscopes << subscope
-                end
-              end
-              [server, subscopes]
+              [server, server.shards.merge(scope)]
             end]
           else
             scopes = scope.group_by(&:database_server)
-            if parallel > 1
-              parallel = [(max_procs.to_f / scopes.count).ceil, parallel].min if max_procs
-              scopes = Hash[scopes.map do |(server, shards)|
-                [server, shards.in_groups(parallel, false).compact]
-              end]
-            else
-              scopes = Hash[scopes.map { |(server, shards)| [server, [shards]] }]
-            end
           end
 
           exception_pipes = []
@@ -244,8 +223,8 @@ module Switchman
           end
 
           # only one process; don't bother forking
-          if scopes.length == 1 && parallel == 1
-            return with_each_shard(scopes.first.last.first, categories, options) { yield }
+          if scopes.length == 1
+            return with_each_shard(scopes.first.last, categories, options) { yield }
           end
 
           # clear connections prior to forking (no more queries will be executed in the parent,
@@ -253,84 +232,78 @@ module Switchman
           # silly like dealloc'ing prepared statements)
           ::ActiveRecord::Base.clear_all_connections!
 
-          scopes.each do |server, subscopes|
-            subscopes.each_with_index do |subscope, idx|
-              if subscopes.length > 1
-                name = "#{server.id} #{idx + 1}"
-              else
-                name = server.id
-              end
+          scopes.each do |server, subscope|
+            name = server.id
+
+            exception_pipe = IO.pipe
+            exception_pipes << exception_pipe
+            pid, io_in, io_out, io_err = Open4.pfork4(lambda do
+              begin
+                Switchman.config[:on_fork_proc]&.call
+
+                # set a pretty name for the process title, up to 128 characters
+                # (we don't actually know the limit, depending on how the process
+                # was started)
+                # first, simplify the binary name by stripping directories,
+                # then truncate arguments as necessary
+                bin = File.basename($0)  # Process.argv0 doesn't work on Ruby 2.5 (https://bugs.ruby-lang.org/issues/15887)
+                max_length = 128 - bin.length - name.length - 3
+                args = ARGV.join(" ")
+                if max_length >= 0
+                  args = args[0..max_length]
+                end
+                new_title = [bin, args, name].join(" ")
+                Process.setproctitle(new_title)
 
-              exception_pipe = IO.pipe
-              exception_pipes << exception_pipe
-              pid, io_in, io_out, io_err = Open4.pfork4(lambda do
+                with_each_shard(subscope, categories, options) { yield }
+                exception_pipe.last.close
+              rescue Exception => e
                 begin
-                  Switchman.config[:on_fork_proc]&.call
-
-                  # set a pretty name for the process title, up to 128 characters
-                  # (we don't actually know the limit, depending on how the process
-                  # was started)
-                  # first, simplify the binary name by stripping directories,
-                  # then truncate arguments as necessary
-                  bin = File.basename($0)  # Process.argv0 doesn't work on Ruby 2.5 (https://bugs.ruby-lang.org/issues/15887)
-                  max_length = 128 - bin.length - name.length - 3
-                  args = ARGV.join(" ")
-                  if max_length >= 0
-                    args = args[0..max_length]
-                  end
-                  new_title = [bin, args, name].join(" ")
-                  Process.setproctitle(new_title)
-
-                  with_each_shard(subscope, categories, options) { yield }
-                  exception_pipe.last.close
-                rescue Exception => e
-                  begin
-                    dumped = Marshal.dump(e)
-                    dumped = nil if dumped.length > 64 * 1024
-                  rescue
-                    dumped = nil
-                  end
+                  dumped = Marshal.dump(e)
+                  dumped = nil if dumped.length > 64 * 1024
+                rescue
+                  dumped = nil
+                end
 
-                  if dumped.nil?
-                    # couldn't dump the exception; create a copy with just
-                    # the message and the backtrace
-                    e2 = e.class.new(e.message)
-                    backtrace = e.backtrace
-                    # truncate excessively long backtraces
-                    if backtrace.length > 50
-                      backtrace = backtrace[0...25] + ['...'] + backtrace[-25..-1]
-                    end
-                    e2.set_backtrace(backtrace)
-                    e2.instance_variable_set(:@active_shards, e.instance_variable_get(:@active_shards))
-                    dumped = Marshal.dump(e2)
+                if dumped.nil?
+                  # couldn't dump the exception; create a copy with just
+                  # the message and the backtrace
+                  e2 = e.class.new(e.message)
+                  backtrace = e.backtrace
+                  # truncate excessively long backtraces
+                  if backtrace.length > 50
+                    backtrace = backtrace[0...25] + ['...'] + backtrace[-25..-1]
                   end
-
-                  exception_pipe.last.set_encoding(dumped.encoding)
-                  exception_pipe.last.write(dumped)
-                  exception_pipe.last.flush
-                  exception_pipe.last.close
-                  exit! 1
+                  e2.set_backtrace(backtrace)
+                  e2.instance_variable_set(:@active_shards, e.instance_variable_get(:@active_shards))
+                  dumped = Marshal.dump(e2)
                 end
-              end)
-              exception_pipe.last.close
-              pids << pid
-              io_in.close # don't care about writing to stdin
-              out_fds << io_out
-              err_fds << io_err
-              pid_to_name_map[pid] = name
-              fd_to_name_map[io_out] = name
-              fd_to_name_map[io_err] = name
-
-              while max_procs && pids.count >= max_procs
-                while max_procs && out_fds.count >= max_procs
-                  # wait for output if we've hit the max_procs limit
-                  wait_for_output.call(out_fds, err_fds, fd_to_name_map)
-                end
-                # we've gotten all the output from one fd so wait for its child process to exit
-                found_pid, status = Process.wait2
-                pids.delete(found_pid)
-                errors << pid_to_name_map[found_pid] if status.exitstatus != 0
+
+                exception_pipe.last.set_encoding(dumped.encoding)
+                exception_pipe.last.write(dumped)
+                exception_pipe.last.flush
+                exception_pipe.last.close
+                exit! 1
+              end
+            end)
+            exception_pipe.last.close
+            pids << pid
+            io_in.close # don't care about writing to stdin
+            out_fds << io_out
+            err_fds << io_err
+            pid_to_name_map[pid] = name
+            fd_to_name_map[io_out] = name
+            fd_to_name_map[io_err] = name
+
+            while pids.count >= parallel
+              while out_fds.count >= parallel
+                # wait for output if we've hit the parallel limit
+                wait_for_output.call(out_fds, err_fds, fd_to_name_map)
               end
+              # we've gotten all the output from one fd so wait for its child process to exit
+              found_pid, status = Process.wait2
+              pids.delete(found_pid)
+              errors << pid_to_name_map[found_pid] if status.exitstatus != 0
             end
           end
 
@@ -544,24 +517,6 @@ module Switchman
         shard || source_shard || Shard.current
       end
 
-      # given the provided option, determines whether we need to (and whether
-      # it's possible) to determine a reasonable default.
-      def determine_max_procs(max_procs_input, parallel_input=2)
-        max_procs = nil
-        if max_procs_input
-          max_procs = max_procs_input.to_i
-          max_procs = nil if max_procs == 0
-        else
-          return 1 if parallel_input.nil? || parallel_input < 1
-          cpus = Environment.cpu_count
-          if cpus && cpus > 0
-            max_procs = cpus * parallel_input
-          end
-        end
-
-        return max_procs
-      end
-
       private
       # in-process caching
       def cached_shards

data/lib/switchman/active_record/connection_pool.rb

@@ -75,7 +75,9 @@ module Switchman
           end
           @available.clear
           @connections.each do |conn|
-            @available.add conn
+            # we do not want to be introducing leased
+            # connections into the available queue
+            @available.add(conn) unless conn.in_use?
           end
         end
       end

data/lib/switchman/active_record/migration.rb

@@ -30,10 +30,13 @@ module Switchman
     end
 
     module Migrator
-      # significant change: hash shard id, not database name
+      # significant change: just return MIGRATOR_SALT directly
+      # especially if you're going through pgbouncer, the database
+      # name you're accessing may not be consistent. it is NOT allowed
+      # to run migrations against multiple shards in the same database
+      # concurrently
       def generate_migrator_advisory_lock_id
-        shard_name_hash = Zlib.crc32(Shard.current.name)
-        ::ActiveRecord::Migrator::MIGRATOR_SALT * shard_name_hash
+        ::ActiveRecord::Migrator::MIGRATOR_SALT
       end
 
       if ::Rails.version >= '6.0'

data/lib/switchman/active_record/persistence.rb

@@ -13,6 +13,15 @@ module Switchman
           shard.activate(self.class.shard_category) { super }
         end
       end
+
+      def delete
+        db = shard.database_server
+        if ::GuardRail.environment != db.guard_rail_environment
+          return db.unguard { super }
+        else
+          super
+        end
+      end
     end
   end
-end
+end

data/lib/switchman/active_record/postgresql_adapter.rb

@@ -102,7 +102,7 @@ module Switchman
             WHERE i.relkind = 'i'
               AND d.indisprimary = 'f'
               AND t.relname = '#{table_name}'
-              AND i.relnamespace IN (SELECT oid FROM pg_namespace WHERE nspname = '#{shard.name}' )
+              AND t.relnamespace IN (SELECT oid FROM pg_namespace WHERE nspname = '#{shard.name}' )
             ORDER BY i.relname
           SQL
 

data/lib/switchman/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Switchman
-  VERSION = "2.1.5"
+  VERSION = "2.2.2"
 end

data/lib/tasks/switchman.rake

@@ -46,7 +46,8 @@ module Switchman
     end
 
     def self.options
-      { parallel: ENV['PARALLEL'].to_i, max_procs: ENV['MAX_PARALLEL_PROCS'] }
+      # we still pass through both of these options for back-compat purposes
+      { parallel: ENV['PARALLEL']&.to_i, max_procs: ENV['MAX_PARALLEL_PROCS']&.to_i }
     end
 
     # categories - an array or proc, to activate as the current shard during the
@@ -89,7 +90,7 @@ module Switchman
                 nil
               end
             rescue => e
-              puts "Exception from #{e.current_shard.id}: #{e.current_shard.description}" if options[:parallel] != 0
+              puts "Exception from #{e.current_shard.id}: #{e.current_shard.description}" if options[:parallel].to_i != 0
               raise
             end
           end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: switchman
 version: !ruby/object:Gem::Version
-  version: 2.1.5
+  version: 2.2.2
 platform: ruby
 authors:
 - Cody Cutrer
@@ -10,7 +10,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2021-08-03 00:00:00.000000000 Z
+date: 2021-10-19 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: railties