switchman 2.1.4 → 2.2.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: d96c8878af2c919433712e502ab527fd77e811eeb88a652357cc004fb5a40ac7
-  data.tar.gz: b8f9e1cb90e7dfbfb25ea02a8699a6c8e20d305f60a13f062a1d71d1da27816e
+  metadata.gz: 34560d2af725df11f4d5f35c512ee53f5e0ad4b614e7caf3d8558895c754c096
+  data.tar.gz: ab3bdc95958fe37f8210dd853e363394bf564c5600c4750cdb2cba408515a5dc
 SHA512:
-  metadata.gz: bb9130947cbff85fef69fdb3f3ac54c29aa9cf7f95507faf92ac97673828addeebe58c9ff1fc478ff34daa834e2f42774a81a99b5bd42dde7e06e3fd63ff6dca
-  data.tar.gz: 1e9fd256868e5b35a0f8d34c5726af34bc7381d76f77f9988d623da84b510959a4ff8563956c797c4921961a8540dd1f8fe74882df9027b4946b93e087309284
+  metadata.gz: 1b703818c43d4bdf7f39bb628e80b8d088aedb3b860e383339a99015da2d9780f78fa3eabd0b3e7f813681388f72155979e9de13eb1e2bdefcfeb9a5298d42a1
+  data.tar.gz: 5ea941cdbb24705af946fe34c583ee323ae5e8a9ac92808c67b8d6972c3ae52b04a3e4e724f824b2a9fef17ce2e1be9a6791498e3922ddea59a16933d9aa0fda

data/app/models/switchman/shard.rb

@@ -129,6 +129,11 @@ module Switchman
         cached_shards[id]
       end
 
+      def preload_cache
+        cached_shards.reverse_merge!(active_shards.values.index_by(&:id))
+        cached_shards.reverse_merge!(all.index_by(&:id))
+      end
+
       def clear_cache
         cached_shards.clear
       end
@@ -139,10 +144,9 @@ module Switchman
       # * +categories+ - an array of categories to activate
       # * +options+ -
       #    :parallel - true/false to execute in parallel, or a integer of how many
-      #                sub-processes per database server. Note that parallel
-      #                invocation currently uses forking, so should be used sparingly
-      #                because errors are not raised, and you cannot get results back
-      #    :max_procs - only run this many parallel processes at a time
+      #                sub-processes. Note that parallel invocation currently uses
+      #                forking, so should be used sparingly because you cannot get
+      #                results back
       #    :exception - :ignore, :raise, :defer (wait until the end and raise the first
       #                error), or a proc
       def with_each_shard(*args)
@@ -163,11 +167,14 @@ module Switchman
           scope, categories = args
         end
 
+        # back-compat
+        options[:parallel] = options.delete(:max_procs) if options.key?(:max_procs)
+
         parallel = case options[:parallel]
                      when true
-                       1
+                       [Environment.cpu_count || 2, 2].min
                      when false, nil
-                       0
+                       1
                      else
                        options[:parallel]
                    end
@@ -178,47 +185,19 @@ module Switchman
           scope = scope.order(::Arel.sql("database_server_id IS NOT NULL, database_server_id, id"))
         end
 
-        if parallel > 0
-          max_procs = determine_max_procs(options.delete(:max_procs), parallel)
+        if parallel > 1
           if ::ActiveRecord::Relation === scope
             # still need a post-uniq, cause the default database server could be NULL or Rails.env in the db
             database_servers = scope.reorder('database_server_id').select(:database_server_id).distinct.
                 map(&:database_server).compact.uniq
             # nothing to do
             return if database_servers.count == 0
-            parallel = [(max_procs.to_f / database_servers.count).ceil, parallel].min if max_procs
 
             scopes = Hash[database_servers.map do |server|
-              server_scope = server.shards.merge(scope)
-              if parallel == 1
-                subscopes = [server_scope]
-              else
-                subscopes = []
-                total = server_scope.count
-                ranges = []
-                server_scope.find_ids_in_ranges(:batch_size => (total.to_f / parallel).ceil) do |min, max|
-                  ranges << [min, max]
-                end
-                # create a half-open range on the last one
-                ranges.last[1] = nil
-                ranges.each do |min, max|
-                  subscope = server_scope.where("id>=?", min)
-                  subscope = subscope.where("id<=?", max) if max
-                  subscopes << subscope
-                end
-              end
-              [server, subscopes]
+              [server, server.shards.merge(scope)]
             end]
           else
             scopes = scope.group_by(&:database_server)
-            if parallel > 1
-              parallel = [(max_procs.to_f / scopes.count).ceil, parallel].min if max_procs
-              scopes = Hash[scopes.map do |(server, shards)|
-                [server, shards.in_groups(parallel, false).compact]
-              end]
-            else
-              scopes = Hash[scopes.map { |(server, shards)| [server, [shards]] }]
-            end
           end
 
           exception_pipes = []
@@ -244,8 +223,8 @@ module Switchman
           end
 
           # only one process; don't bother forking
-          if scopes.length == 1 && parallel == 1
-            return with_each_shard(scopes.first.last.first, categories, options) { yield }
+          if scopes.length == 1
+            return with_each_shard(scopes.first.last, categories, options) { yield }
           end
 
           # clear connections prior to forking (no more queries will be executed in the parent,
@@ -253,84 +232,78 @@ module Switchman
           # silly like dealloc'ing prepared statements)
           ::ActiveRecord::Base.clear_all_connections!
 
-          scopes.each do |server, subscopes|
-            subscopes.each_with_index do |subscope, idx|
-              if subscopes.length > 1
-                name = "#{server.id} #{idx + 1}"
-              else
-                name = server.id
-              end
+          scopes.each do |server, subscope|
+            name = server.id
+
+            exception_pipe = IO.pipe
+            exception_pipes << exception_pipe
+            pid, io_in, io_out, io_err = Open4.pfork4(lambda do
+              begin
+                Switchman.config[:on_fork_proc]&.call
+
+                # set a pretty name for the process title, up to 128 characters
+                # (we don't actually know the limit, depending on how the process
+                # was started)
+                # first, simplify the binary name by stripping directories,
+                # then truncate arguments as necessary
+                bin = File.basename($0)  # Process.argv0 doesn't work on Ruby 2.5 (https://bugs.ruby-lang.org/issues/15887)
+                max_length = 128 - bin.length - name.length - 3
+                args = ARGV.join(" ")
+                if max_length >= 0
+                  args = args[0..max_length]
+                end
+                new_title = [bin, args, name].join(" ")
+                Process.setproctitle(new_title)
 
-              exception_pipe = IO.pipe
-              exception_pipes << exception_pipe
-              pid, io_in, io_out, io_err = Open4.pfork4(lambda do
+                with_each_shard(subscope, categories, options) { yield }
+                exception_pipe.last.close
+              rescue Exception => e
                 begin
-                  Switchman.config[:on_fork_proc]&.call
-
-                  # set a pretty name for the process title, up to 128 characters
-                  # (we don't actually know the limit, depending on how the process
-                  # was started)
-                  # first, simplify the binary name by stripping directories,
-                  # then truncate arguments as necessary
-                  bin = File.basename($0)  # Process.argv0 doesn't work on Ruby 2.5 (https://bugs.ruby-lang.org/issues/15887)
-                  max_length = 128 - bin.length - name.length - 3
-                  args = ARGV.join(" ")
-                  if max_length >= 0
-                    args = args[0..max_length]
-                  end
-                  new_title = [bin, args, name].join(" ")
-                  Process.setproctitle(new_title)
-
-                  with_each_shard(subscope, categories, options) { yield }
-                  exception_pipe.last.close
-                rescue Exception => e
-                  begin
-                    dumped = Marshal.dump(e)
-                    dumped = nil if dumped.length > 64 * 1024
-                  rescue
-                    dumped = nil
-                  end
+                  dumped = Marshal.dump(e)
+                  dumped = nil if dumped.length > 64 * 1024
+                rescue
+                  dumped = nil
+                end
 
-                  if dumped.nil?
-                    # couldn't dump the exception; create a copy with just
-                    # the message and the backtrace
-                    e2 = e.class.new(e.message)
-                    backtrace = e.backtrace
-                    # truncate excessively long backtraces
-                    if backtrace.length > 50
-                      backtrace = backtrace[0...25] + ['...'] + backtrace[-25..-1]
-                    end
-                    e2.set_backtrace(backtrace)
-                    e2.instance_variable_set(:@active_shards, e.instance_variable_get(:@active_shards))
-                    dumped = Marshal.dump(e2)
+                if dumped.nil?
+                  # couldn't dump the exception; create a copy with just
+                  # the message and the backtrace
+                  e2 = e.class.new(e.message)
+                  backtrace = e.backtrace
+                  # truncate excessively long backtraces
+                  if backtrace.length > 50
+                    backtrace = backtrace[0...25] + ['...'] + backtrace[-25..-1]
                   end
-
-                  exception_pipe.last.set_encoding(dumped.encoding)
-                  exception_pipe.last.write(dumped)
-                  exception_pipe.last.flush
-                  exception_pipe.last.close
-                  exit! 1
+                  e2.set_backtrace(backtrace)
+                  e2.instance_variable_set(:@active_shards, e.instance_variable_get(:@active_shards))
+                  dumped = Marshal.dump(e2)
                 end
-              end)
-              exception_pipe.last.close
-              pids << pid
-              io_in.close # don't care about writing to stdin
-              out_fds << io_out
-              err_fds << io_err
-              pid_to_name_map[pid] = name
-              fd_to_name_map[io_out] = name
-              fd_to_name_map[io_err] = name
-
-              while max_procs && pids.count >= max_procs
-                while max_procs && out_fds.count >= max_procs
-                  # wait for output if we've hit the max_procs limit
-                  wait_for_output.call(out_fds, err_fds, fd_to_name_map)
-                end
-                # we've gotten all the output from one fd so wait for its child process to exit
-                found_pid, status = Process.wait2
-                pids.delete(found_pid)
-                errors << pid_to_name_map[found_pid] if status.exitstatus != 0
+
+                exception_pipe.last.set_encoding(dumped.encoding)
+                exception_pipe.last.write(dumped)
+                exception_pipe.last.flush
+                exception_pipe.last.close
+                exit! 1
+              end
+            end)
+            exception_pipe.last.close
+            pids << pid
+            io_in.close # don't care about writing to stdin
+            out_fds << io_out
+            err_fds << io_err
+            pid_to_name_map[pid] = name
+            fd_to_name_map[io_out] = name
+            fd_to_name_map[io_err] = name
+
+            while pids.count >= parallel
+              while out_fds.count >= parallel
+                # wait for output if we've hit the parallel limit
+                wait_for_output.call(out_fds, err_fds, fd_to_name_map)
               end
+              # we've gotten all the output from one fd so wait for its child process to exit
+              found_pid, status = Process.wait2
+              pids.delete(found_pid)
+              errors << pid_to_name_map[found_pid] if status.exitstatus != 0
             end
           end
 
@@ -544,24 +517,6 @@ module Switchman
         shard || source_shard || Shard.current
       end
 
-      # given the provided option, determines whether we need to (and whether
-      # it's possible) to determine a reasonable default.
-      def determine_max_procs(max_procs_input, parallel_input=2)
-        max_procs = nil
-        if max_procs_input
-          max_procs = max_procs_input.to_i
-          max_procs = nil if max_procs == 0
-        else
-          return 1 if parallel_input.nil? || parallel_input < 1
-          cpus = Environment.cpu_count
-          if cpus && cpus > 0
-            max_procs = cpus * parallel_input
-          end
-        end
-
-        return max_procs
-      end
-
       private
       # in-process caching
       def cached_shards

data/lib/switchman/active_record/base.rb

@@ -158,7 +158,7 @@ module Switchman
       end
 
       def update_columns(*)
-        db = Shard.current(self.class.shard_category).database_server
+        db = shard.database_server
         if ::GuardRail.environment != db.guard_rail_environment
           return db.unguard { super }
         else

data/lib/switchman/active_record/migration.rb

@@ -30,10 +30,13 @@ module Switchman
     end
 
     module Migrator
-      # significant change: hash shard id, not database name
+      # significant change: just return MIGRATOR_SALT directly
+      # especially if you're going through pgbouncer, the database
+      # name you're accessing may not be consistent. it is NOT allowed
+      # to run migrations against multiple shards in the same database
+      # concurrently
       def generate_migrator_advisory_lock_id
-        shard_name_hash = Zlib.crc32(Shard.current.name)
-        ::ActiveRecord::Migrator::MIGRATOR_SALT * shard_name_hash
+        ::ActiveRecord::Migrator::MIGRATOR_SALT
       end
 
       if ::Rails.version >= '6.0'

data/lib/switchman/active_record/persistence.rb

@@ -13,6 +13,15 @@ module Switchman
           shard.activate(self.class.shard_category) { super }
         end
       end
+
+      def delete
+        db = shard.database_server
+        if ::GuardRail.environment != db.guard_rail_environment
+          return db.unguard { super }
+        else
+          super
+        end
+      end
     end
   end
-end
+end

data/lib/switchman/active_record/postgresql_adapter.rb

@@ -102,7 +102,7 @@ module Switchman
             WHERE i.relkind = 'i'
               AND d.indisprimary = 'f'
               AND t.relname = '#{table_name}'
-              AND i.relnamespace IN (SELECT oid FROM pg_namespace WHERE nspname = '#{shard.name}' )
+              AND t.relnamespace IN (SELECT oid FROM pg_namespace WHERE nspname = '#{shard.name}' )
             ORDER BY i.relname
           SQL
 

data/lib/switchman/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Switchman
-  VERSION = "2.1.4"
+  VERSION = "2.2.1"
 end

data/lib/tasks/switchman.rake

@@ -46,7 +46,8 @@ module Switchman
     end
 
     def self.options
-      { parallel: ENV['PARALLEL'].to_i, max_procs: ENV['MAX_PARALLEL_PROCS'] }
+      # we still pass through both of these options for back-compat purposes
+      { parallel: ENV['PARALLEL']&.to_i, max_procs: ENV['MAX_PARALLEL_PROCS']&.to_i }
     end
 
     # categories - an array or proc, to activate as the current shard during the
@@ -89,7 +90,7 @@ module Switchman
                 nil
               end
             rescue => e
-              puts "Exception from #{e.current_shard.id}: #{e.current_shard.description}" if options[:parallel] != 0
+              puts "Exception from #{e.current_shard.id}: #{e.current_shard.description}" if options[:parallel].to_i != 0
               raise
             end
           end

metadata

@@ -1,16 +1,16 @@
 --- !ruby/object:Gem::Specification
 name: switchman
 version: !ruby/object:Gem::Version
-  version: 2.1.4
+  version: 2.2.1
 platform: ruby
 authors:
 - Cody Cutrer
 - James Williams
 - Jacob Fugal
-autorequire: 
+autorequire:
 bindir: bin
 cert_chain: []
-date: 2021-07-29 00:00:00.000000000 Z
+date: 2021-08-25 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: railties
@@ -257,7 +257,7 @@ homepage: http://www.instructure.com/
 licenses:
 - MIT
 metadata: {}
-post_install_message: 
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -272,8 +272,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.0.3
-signing_key: 
+rubygems_version: 3.2.24
+signing_key:
 specification_version: 4
 summary: Rails sharding magic
 test_files: []