switchman 3.0.6 → 3.1.0

data/{app/models → lib}/switchman/shard.rb

@@ -1,10 +1,5 @@
 # frozen_string_literal: true
 
-require 'switchman/database_server'
-require 'switchman/default_shard'
-require 'switchman/environment'
-require 'switchman/errors'
-
 module Switchman
   class Shard < UnshardedRecord
     # ten trillion possible ids per shard. yup.
@@ -42,14 +37,9 @@ module Switchman
           # Now find the actual record, if it exists
           @default = begin
             find_cached('default_shard') { Shard.where(default: true).take } || default
-          # If we are *super* early in boot, the connection pool won't exist; we don't want to fill in the default shard yet
-          rescue ::ActiveRecord::ConnectionNotEstablished
-            nil
-          # rescue the fake default if the table doesn't exist
           rescue
             default
           end
-          return default unless @default
 
           # make sure this is not erroneously cached
           @default.database_server.remove_instance_variable(:@primary_shard) if @default.database_server.instance_variable_defined?(:@primary_shard)
@@ -127,8 +117,7 @@ module Switchman
       # * +classes+ - an array of classes to activate
       #    parallel: - true/false to execute in parallel, or an integer of how many
       #                sub-processes. Note that parallel invocation currently uses
-      #                forking, so should be used sparingly because you cannot get
-      #                results back
+      #                forking.
       #    exception: - :ignore, :raise, :defer (wait until the end and raise the first
       #                error), or a proc
       def with_each_shard(*args, parallel: false, exception: :raise, &block)
@@ -160,138 +149,48 @@ module Switchman
             # nothing to do
             return if database_servers.count.zero?
 
-            scopes = database_servers.map do |server|
-              [server, server.shards.merge(scope)]
-            end.to_h
+            scopes = database_servers.to_h do |server|
+              [server, scope.merge(server.shards)]
+            end
           else
             scopes = scope.group_by(&:database_server)
           end
 
-          exception_pipes = []
-          pids = []
-          out_fds = []
-          err_fds = []
-          pid_to_name_map = {}
-          fd_to_name_map = {}
-          errors = []
-
-          wait_for_output = lambda do
-            ready, = IO.select(out_fds + err_fds)
-            ready.each do |fd|
-              if fd.eof?
-                fd.close
-                out_fds.delete(fd)
-                err_fds.delete(fd)
-                next
-              end
-              line = fd.readline
-              puts "#{fd_to_name_map[fd]}: #{line}"
-            end
-          end
-
-          # only one process; don't bother forking
-          return with_each_shard(scopes.first.last, classes, exception: exception, &block) if scopes.length == 1
-
           # clear connections prior to forking (no more queries will be executed in the parent,
           # and we want them gone so that we don't accidentally use them post-fork doing something
           # silly like dealloc'ing prepared statements)
           ::ActiveRecord::Base.clear_all_connections!
 
-          scopes.each do |server, subscope|
+          parent_process_name = `ps -ocommand= -p#{Process.pid}`.slice(/#{$0}.*/)
+          ret = ::Parallel.map(scopes, in_processes: scopes.length > 1 ? parallel : 0) do |server, subscope|
             name = server.id
-
-            exception_pipe = IO.pipe
-            exception_pipes << exception_pipe
-            pid, io_in, io_out, io_err = Open4.pfork4(lambda do
-              Switchman.config[:on_fork_proc]&.call
-
-              # set a pretty name for the process title, up to 128 characters
-              # (we don't actually know the limit, depending on how the process
-              # was started)
-              # first, simplify the binary name by stripping directories,
-              # then truncate arguments as necessary
-              bin = File.basename($0) # Process.argv0 doesn't work on Ruby 2.5 (https://bugs.ruby-lang.org/issues/15887)
-              max_length = 128 - bin.length - name.length - 3
-              args = ARGV.join(' ')
-              args = args[0..max_length] if max_length >= 0
-              new_title = [bin, args, name].join(' ')
+            # rubocop:disable Style/GlobalStdStream
+            $stdout = Parallel::PrefixingIO.new(name, STDOUT)
+            $stderr = Parallel::PrefixingIO.new(name, STDERR)
+            # rubocop:enable Style/GlobalStdStream
+            begin
+              max_length = 128 - name.length - 3
+              short_parent_name = parent_process_name[0..max_length] if max_length >= 0
+              new_title = [short_parent_name, name].join(' ')
               Process.setproctitle(new_title)
-
-              with_each_shard(subscope, classes, exception: exception, &block)
-              exception_pipe.last.close
-            rescue Exception => e # rubocop:disable Lint/RescueException
-              begin
-                dumped = Marshal.dump(e)
-                dumped = nil if dumped.length > 64 * 1024
-              rescue
-                dumped = nil
-              end
-
-              if dumped.nil?
-                # couldn't dump the exception; create a copy with just
-                # the message and the backtrace
-                e2 = e.class.new(e.message)
-                backtrace = e.backtrace
-                # truncate excessively long backtraces
-                backtrace = backtrace[0...25] + ['...'] + backtrace[-25..] if backtrace.length > 50
-                e2.set_backtrace(backtrace)
-                e2.instance_variable_set(:@active_shards, e.instance_variable_get(:@active_shards))
-                dumped = Marshal.dump(e2)
-              end
-              exception_pipe.last.set_encoding(dumped.encoding)
-              exception_pipe.last.write(dumped)
-              exception_pipe.last.flush
-              exception_pipe.last.close
-              exit! 1
-            end)
-            exception_pipe.last.close
-            pids << pid
-            io_in.close # don't care about writing to stdin
-            out_fds << io_out
-            err_fds << io_err
-            pid_to_name_map[pid] = name
-            fd_to_name_map[io_out] = name
-            fd_to_name_map[io_err] = name
-
-            while pids.count >= parallel
-              while out_fds.count >= parallel
-                # wait for output if we've hit the parallel limit
-                wait_for_output.call
-              end
-              # we've gotten all the output from one fd so wait for its child process to exit
-              found_pid, status = Process.wait2
-              pids.delete(found_pid)
-              errors << pid_to_name_map[found_pid] if status.exitstatus != 0
+              Switchman.config[:on_fork_proc]&.call
+              with_each_shard(subscope, classes, exception: exception, &block).map { |result| Parallel::ResultWrapper.new(result) }
+            rescue => e
+              logger.error e.full_message
+              Parallel::QuietExceptionWrapper.new(name, ::Parallel::ExceptionWrapper.new(e))
             end
-            # we've gotten all the output from one fd so wait for its child process to exit
-            found_pid, status = Process.wait2
-            pids.delete(found_pid)
-            errors << pid_to_name_map[found_pid] if status.exitstatus != 0
-          end
-
-          wait_for_output.call while out_fds.any? || err_fds.any?
-          pids.each do |pid|
-            _, status = Process.waitpid2(pid)
-            errors << pid_to_name_map[pid] if status.exitstatus != 0
-          end
-
-          # check for an exception; we only re-raise the first one
-          exception_pipes.each do |exception_pipe|
-            serialized_exception = exception_pipe.first.read
-            next if serialized_exception.empty?
-
-            ex = Marshal.load(serialized_exception) # rubocop:disable Security/MarshalLoad
-            raise ex
-          ensure
-            exception_pipe.first.close
-          end
+          end.flatten
 
+          errors = ret.select { |val| val.is_a?(Parallel::QuietExceptionWrapper) }
           unless errors.empty?
-            raise ParallelShardExecError,
-                  "The following subprocesses did not exit cleanly: #{errors.sort.join(', ')}"
+            raise errors.first.exception if errors.length == 1
+
+            raise Errors::ParallelShardExecError,
+                  "The following database server(s) did not finish processing cleanly: #{errors.map(&:name).sort.join(', ')}",
+                  cause: errors.first.exception
           end
 
-          return
+          return ret.map(&:result)
         end
 
         classes ||= []
@@ -471,11 +370,11 @@ module Switchman
 
       def add_sharded_model(klass)
         @sharded_models = (sharded_models + [klass]).freeze
-        initialize_sharding
+        configure_connects_to
       end
 
-      def initialize_sharding
-        full_connects_to_hash = DatabaseServer.all.map { |db| [db.id.to_sym, db.connects_to_hash] }.to_h
+      def configure_connects_to
+        full_connects_to_hash = DatabaseServer.all.to_h { |db| [db.id.to_sym, db.connects_to_hash] }
         sharded_models.each do |klass|
           connects_to_hash = full_connects_to_hash.deep_dup
           if klass == UnshardedRecord

data/lib/switchman/sharded_instrumenter.rb

@@ -16,7 +16,7 @@ module Switchman
         payload[:shard] = {
           database_server_id: shard.database_server.id,
           id: shard.id,
-          env: shard.database_server.guard_rail_environment
+          env: ::Rails.version < '7.0' ? @shard_host.pool.connection_klass&.current_role : @shard_host.pool.connection_class&.current_role
         }
       end
       super name, payload

data/lib/switchman/standard_error.rb

@@ -3,20 +3,19 @@
 module Switchman
   module StandardError
     def initialize(*args)
-      # Shard.current can throw this when switchman isn't working right; if we try to
-      # do our stuff here, it'll cause a SystemStackError, which is a pain to deal with
-      if is_a?(::ActiveRecord::ConnectionNotEstablished)
-        super
-        return
-      end
+      super
+      # These seem to get themselves into a bad state if we try to lookup shards while processing
+      return if is_a?(IO::EAGAINWaitReadable)
+
+      return if Thread.current[:switchman_error_handler]
 
       begin
-        @active_shards = Shard.active_shards if defined?(Shard)
-      rescue ::ActiveRecord::ConnectionNotEstablished
-        # If we hit an error really early in boot, activerecord may not be initialized yet
-      end
+        Thread.current[:switchman_error_handler] = true
 
-      super
+        @active_shards ||= Shard.active_shards
+      ensure
+        Thread.current[:switchman_error_handler] = nil
+      end
     end
 
     def current_shard(klass = ::ActiveRecord::Base)

data/{app/models → lib}/switchman/unsharded_record.rb

@@ -2,6 +2,6 @@
 
 module Switchman
   class UnshardedRecord < ::ActiveRecord::Base
-    sharded_model
+    self.abstract_class = true
   end
 end

data/lib/switchman/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Switchman
-  VERSION = '3.0.6'
+  VERSION = '3.1.0'
 end

data/lib/switchman.rb

@@ -1,8 +1,21 @@
 # frozen_string_literal: true
 
 require 'guard_rail'
-require 'switchman/open4'
-require 'switchman/engine'
+require 'zeitwerk'
+
+class SwitchmanInflector < Zeitwerk::GemInflector
+  def camelize(basename, abspath)
+    if basename =~ /\Apostgresql_(.*)/
+      'PostgreSQL' + super($1, abspath)
+    else
+      super
+    end
+  end
+end
+
+loader = Zeitwerk::Loader.for_gem
+loader.inflector = SwitchmanInflector.new(__FILE__)
+loader.setup
 
 module Switchman
   def self.config
@@ -18,5 +31,12 @@ module Switchman
     @cache = cache
   end
 
+  def self.foreign_key_check(name, type, limit: nil)
+    puts "WARNING: All foreign keys need to be 8-byte integers. #{name} looks like a foreign key. If so, please add the option: `:limit => 8`" if name.to_s =~ /_id\z/ && type.to_s == 'integer' && limit.to_i < 8
+  end
+
   class OrderOnMultiShardQuery < RuntimeError; end
 end
+
+# Load the engine and everything associated at gem load time
+Switchman::Engine

data/lib/tasks/switchman.rake

@@ -1,5 +1,16 @@
 # frozen_string_literal: true
 
+# In rails 7.0+ if you have only 1 db in the env it doesn't try to do explicit activation
+# (and for rails purposes we only have one db per env because each database server is a separate env)
+if Rails.version < '7.0'
+  task_prefix = ::Rake::Task.task_defined?('app:db:migrate') ? 'app:db' : 'db'
+  ::Rake::Task["#{task_prefix}:migrate"].clear_actions.enhance do
+    ::ActiveRecord::Tasks::DatabaseTasks.migrate
+    # Ensure this doesn't blow up when running inside the dummy app
+    Rake::Task["#{task_prefix}:_dump"].invoke
+  end
+end
+
 module Switchman
   module Rake
     def self.filter_database_servers(&block)
@@ -35,9 +46,9 @@ module Switchman
 
       scope = base_scope.order(::Arel.sql('database_server_id IS NOT NULL, database_server_id, id'))
       if servers != DatabaseServer.all
-        conditions = ['database_server_id IN (?)', servers.map(&:id)]
-        conditions.first << ' OR database_server_id IS NULL' if servers.include?(Shard.default.database_server)
-        scope = scope.where(conditions)
+        database_server_ids = servers.map(&:id)
+        database_server_ids << nil if servers.include?(Shard.default.database_server)
+        scope = scope.where(database_server_id: database_server_ids)
       end
 
       scope = shard_scope(scope, shard) if shard
@@ -50,9 +61,7 @@ module Switchman
     end
 
     # classes - an array or proc, to activate as the current shard during the
-    # task. tasks which modify the schema may want to pass all categories in
-    # so that schema updates for non-default tables happen against all shards.
-    # this is handled automatically for the default migration tasks, below.
+    # task.
     def self.shardify_task(task_name, classes: [::ActiveRecord::Base])
       old_task = ::Rake::Task[task_name]
       old_actions = old_task.actions.dup
@@ -77,10 +86,8 @@ module Switchman
               nil
             end
           rescue => e
-            puts "Exception from #{e.current_shard.id}: #{e.current_shard.description}" if options[:parallel] != 0
+            warn "Exception from #{e.current_shard.id}: #{e.current_shard.description}:\n#{e.full_message}" if options[:parallel] != 0
             raise
-
-            # ::ActiveRecord::Base.configurations = old_configurations
           end
         end
       end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: switchman
 version: !ruby/object:Gem::Version
-  version: 3.0.6
+  version: 3.1.0
 platform: ruby
 authors:
 - Cody Cutrer
@@ -10,7 +10,7 @@ authors:
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2022-02-11 00:00:00.000000000 Z
+date: 2022-06-02 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activerecord
@@ -21,7 +21,7 @@ dependencies:
         version: 6.1.4
     - - "<"
       - !ruby/object:Gem::Version
-        version: '6.2'
+        version: '7.1'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
@@ -31,35 +31,35 @@ dependencies:
         version: 6.1.4
     - - "<"
       - !ruby/object:Gem::Version
-        version: '6.2'
+        version: '7.1'
 - !ruby/object:Gem::Dependency
   name: guardrail
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 3.0.0
+        version: 3.0.1
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 3.0.0
+        version: 3.0.1
 - !ruby/object:Gem::Dependency
-  name: open4
+  name: parallel
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 1.3.0
+        version: '1.22'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 1.3.0
+        version: '1.22'
 - !ruby/object:Gem::Dependency
   name: railties
   requirement: !ruby/object:Gem::Requirement
@@ -69,7 +69,7 @@ dependencies:
         version: '6.1'
     - - "<"
       - !ruby/object:Gem::Version
-        version: '6.2'
+        version: '7.1'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
@@ -79,7 +79,7 @@ dependencies:
         version: '6.1'
     - - "<"
       - !ruby/object:Gem::Version
-        version: '6.2'
+        version: '7.1'
 - !ruby/object:Gem::Dependency
   name: appraisal
   requirement: !ruby/object:Gem::Requirement
@@ -242,8 +242,6 @@ extensions: []
 extra_rdoc_files: []
 files:
 - Rakefile
-- app/models/switchman/shard.rb
-- app/models/switchman/unsharded_record.rb
 - db/migrate/20130328212039_create_switchman_shards.rb
 - db/migrate/20130328224244_create_default_shard.rb
 - db/migrate/20161206323434_add_back_default_string_limits_switchman.rb
@@ -253,7 +251,7 @@ files:
 - lib/switchman.rb
 - lib/switchman/action_controller/caching.rb
 - lib/switchman/active_record/abstract_adapter.rb
-- lib/switchman/active_record/association.rb
+- lib/switchman/active_record/associations.rb
 - lib/switchman/active_record/attribute_methods.rb
 - lib/switchman/active_record/base.rb
 - lib/switchman/active_record/calculations.rb
@@ -287,18 +285,21 @@ files:
 - lib/switchman/errors.rb
 - lib/switchman/guard_rail.rb
 - lib/switchman/guard_rail/relation.rb
-- lib/switchman/open4.rb
+- lib/switchman/parallel.rb
 - lib/switchman/r_spec_helper.rb
 - lib/switchman/rails.rb
+- lib/switchman/shard.rb
 - lib/switchman/sharded_instrumenter.rb
 - lib/switchman/standard_error.rb
 - lib/switchman/test_helper.rb
+- lib/switchman/unsharded_record.rb
 - lib/switchman/version.rb
 - lib/tasks/switchman.rake
 homepage: http://www.instructure.com/
 licenses:
 - MIT
-metadata: {}
+metadata:
+  rubygems_mfa_required: 'true'
 post_install_message:
 rdoc_options: []
 require_paths:
@@ -307,14 +308,14 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: '2.6'
+      version: '2.7'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.1.4
+rubygems_version: 3.1.6
 signing_key:
 specification_version: 4
 summary: Rails sharding magic