switchman 3.0.2 → 4.2.5

data/{app/models → lib}/switchman/shard.rb

@@ -1,23 +1,61 @@
 # frozen_string_literal: true
 
-require 'switchman/database_server'
-require 'switchman/default_shard'
-require 'switchman/environment'
-require 'switchman/errors'
-
 module Switchman
   class Shard < UnshardedRecord
     # ten trillion possible ids per shard. yup.
     IDS_PER_SHARD = 10_000_000_000_000
 
+    # rubocop:disable Style/SymbolProc -- transforming to a lambda produces "no receiver given"
     # only allow one default
     validates_uniqueness_of :default, if: ->(s) { s.default? }
+    # rubocop:enable Style/SymbolProc
 
     after_save :clear_cache
     after_destroy :clear_cache
 
     scope :primary, -> { where(name: nil).order(:database_server_id, :id).distinct_on(:database_server_id) }
 
+    scope :in_region, (lambda do |region, include_regionless: true|
+      next in_current_region if region.nil?
+
+      dbs_by_region = DatabaseServer.group_by(&:region)
+      db_count_in_this_region = dbs_by_region[region]&.length.to_i
+      db_count_in_this_region += dbs_by_region[nil]&.length.to_i if include_regionless
+      non_existent_database_servers = Shard.send(:non_existent_database_servers)
+      db_count_in_other_regions = DatabaseServer.all.length -
+                                  db_count_in_this_region +
+                                  non_existent_database_servers.length
+
+      dbs_in_this_region = dbs_by_region[region]&.map(&:id) || []
+      dbs_in_this_region += dbs_by_region[nil]&.map(&:id) || [] if include_regionless
+
+      if db_count_in_this_region <= db_count_in_other_regions
+        if dbs_in_this_region.include?(Shard.default.database_server.id)
+          where("database_server_id IN (?) OR database_server_id IS NULL", dbs_in_this_region)
+        else
+          where(database_server_id: dbs_in_this_region)
+        end
+      elsif db_count_in_other_regions.zero?
+        all
+      else
+        dbs_not_in_this_region = DatabaseServer.map(&:id) - dbs_in_this_region + non_existent_database_servers
+        if dbs_in_this_region.include?(Shard.default.database_server.id)
+          where("database_server_id NOT IN (?) OR database_server_id IS NULL", dbs_not_in_this_region)
+        else
+          where.not(database_server_id: dbs_not_in_this_region)
+        end
+      end
+    end)
+
+    scope :in_current_region, (lambda do |include_regionless: true|
+      # sharding isn't set up? maybe we're in tests, or a somehow degraded environment
+      # either way there's only one shard, and we always want to see it
+      return [default] unless default.is_a?(Switchman::Shard)
+      return all if !Switchman.region || DatabaseServer.none?(&:region)
+
+      in_region(Switchman.region, include_regionless:)
+    end)
+
     class << self
       def sharded_models
         @sharded_models ||= [::ActiveRecord::Base, UnshardedRecord].freeze
@@ -39,15 +77,17 @@ module Switchman
           # the first time we need a dummy dummy for re-entrancy to avoid looping on ourselves
           @default ||= default
 
-          # Now find the actual record, if it exists; rescue the fake default if the table doesn't exist
+          # Now find the actual record, if it exists
           @default = begin
-            find_cached('default_shard') { Shard.where(default: true).take } || default
+            find_cached("default_shard") { Shard.where(default: true).take } || default
           rescue
             default
           end
 
           # make sure this is not erroneously cached
-          @default.database_server.remove_instance_variable(:@primary_shard) if @default.database_server.instance_variable_defined?(:@primary_shard)
+          if @default.database_server.instance_variable_defined?(:@primary_shard)
+            @default.database_server.remove_instance_variable(:@primary_shard)
+          end
 
           # and finally, check for cached references to the default shard on the existing connection
           sharded_models.each do |klass|
@@ -59,50 +99,61 @@ module Switchman
 
       def current(klass = ::ActiveRecord::Base)
         klass ||= ::ActiveRecord::Base
-        klass.connection_pool.shard
+        klass.current_switchman_shard
       end
 
       def activate(shards)
         activated_classes = activate!(shards)
         yield
       ensure
-        activated_classes.each do |klass|
-          klass.connection_pool.shard_stack.pop
+        activated_classes&.each do |klass|
           klass.connected_to_stack.pop
         end
       end
 
       def activate!(shards)
-        activated_classes = []
+        activated_classes = nil
         shards.each do |klass, shard|
           next if klass == UnshardedRecord
 
           next unless klass.current_shard != shard.database_server.id.to_sym ||
-                      klass.connection_pool.shard != shard
+                      klass.current_switchman_shard != shard
 
-          activated_classes << klass
-          klass.connected_to_stack << { shard: shard.database_server.id.to_sym, klasses: [klass] }
-          klass.connection_pool.shard_stack << shard
+          (activated_classes ||= []) << klass
+          klass.connected_to_stack << { shard: shard.database_server.id.to_sym,
+                                        klasses: [klass],
+                                        switchman_shard: shard }
         end
         activated_classes
       end
 
+      def active_shards
+        sharded_models.filter_map do |klass|
+          [klass, current(klass)]
+        end.to_h
+      end
+
       def lookup(id)
         id_i = id.to_i
-        return current if id_i == current.id || id == 'self'
-        return default if id_i == default.id || id.nil? || id == 'default'
+        return current if id_i == current.id || id == "self"
+        return default if id_i == default.id || id.nil? || id == "default"
 
         id = id_i
         raise ArgumentError if id.zero?
 
         unless cached_shards.key?(id)
           cached_shards[id] = Shard.default.activate do
-            find_cached(['shard', id]) { find_by(id: id) }
+            find_cached(["shard", id]) { find_by(id:) }
           end
         end
         cached_shards[id]
       end
 
+      def preload_cache
+        cached_shards.reverse_merge!(active_shards.values.index_by(&:id))
+        cached_shards.reverse_merge!(all.index_by(&:id))
+      end
+
       def clear_cache
         cached_shards.clear
       end
@@ -111,14 +162,13 @@ module Switchman
       #
       # * +shards+ - an array or relation of Shards to iterate over
       # * +classes+ - an array of classes to activate
-      #    parallel: - true/false to execute in parallel, or a integer of how many
-      #                sub-processes per database server. Note that parallel
-      #                invocation currently uses forking, so should be used sparingly
-      #                because errors are not raised, and you cannot get results back
-      #    max_procs: - only run this many parallel processes at a time
+      #    parallel: - true/false to execute in parallel, or an integer of how many
+      #                sub-processes. Note that parallel invocation currently uses
+      #                forking.
       #    exception: - :ignore, :raise, :defer (wait until the end and raise the first
       #                error), or a proc
-      def with_each_shard(*args, parallel: false, max_procs: nil, exception: :raise, &block)
+      #    output: - :simple, :decorated (with database_server_id:shard_name), custom lambda transformer
+      def with_each_shard(*args, parallel: false, exception: :raise, output: :simple)
         raise ArgumentError, "wrong number of arguments (#{args.length} for 0...2)" if args.length > 2
 
         return Array.wrap(yield) unless default.is_a?(Shard)
@@ -133,190 +183,94 @@ module Switchman
           scope, classes = args
         end
 
-        parallel = 1 if parallel == true
+        output = if output == :decorated
+                   ->(arg) { "#{Shard.current.description}: #{arg}" }
+                 elsif output == :simple
+                   nil
+                 else
+                   output
+                 end
+        parallel = [Environment.cpu_count || 2, 2].min if parallel == true
         parallel = 0 if parallel == false || parallel.nil?
 
         scope ||= Shard.all
-        scope = scope.order(::Arel.sql('database_server_id IS NOT NULL, database_server_id, id')) if ::ActiveRecord::Relation === scope && scope.order_values.empty?
+        if ::ActiveRecord::Relation === scope && scope.order_values.empty?
+          scope = scope.order(::Arel.sql("database_server_id IS NOT NULL, database_server_id, id"))
+        end
 
-        if parallel.positive?
-          max_procs = determine_max_procs(max_procs, parallel)
+        if parallel > 1
           if ::ActiveRecord::Relation === scope
             # still need a post-uniq, cause the default database server could be NULL or Rails.env in the db
-            database_servers = scope.reorder('database_server_id').select(:database_server_id).distinct.
-                               map(&:database_server).compact.uniq
+            database_servers = scope.reorder("database_server_id").select(:database_server_id).distinct
+                                    .filter_map(&:database_server).uniq
             # nothing to do
-            return if database_servers.count.zero?
-
-            parallel = [(max_procs.to_f / database_servers.count).ceil, parallel].min if max_procs
-
-            scopes = database_servers.map do |server|
-              server_scope = server.shards.merge(scope)
-              if parallel == 1
-                subscopes = [server_scope]
-              else
-                subscopes = []
-                total = server_scope.count
-                ranges = []
-                server_scope.find_ids_in_ranges(batch_size: (total.to_f / parallel).ceil) do |min, max|
-                  ranges << [min, max]
-                end
-                # create a half-open range on the last one
-                ranges.last[1] = nil
-                ranges.each do |min, max|
-                  subscope = server_scope.where('id>=?', min)
-                  subscope = subscope.where('id<=?', max) if max
-                  subscopes << subscope
-                end
-              end
-              [server, subscopes]
-            end.to_h
-          else
-            scopes = scope.group_by(&:database_server)
-            if parallel > 1
-              parallel = [(max_procs.to_f / scopes.count).ceil, parallel].min if max_procs
-              scopes = scopes.map do |(server, shards)|
-                [server, shards.in_groups(parallel, false).compact]
-              end.to_h
-            else
-              scopes = scopes.map { |(server, shards)| [server, [shards]] }.to_h
-            end
-          end
+            return if database_servers.none?
 
-          exception_pipes = []
-          pids = []
-          out_fds = []
-          err_fds = []
-          pid_to_name_map = {}
-          fd_to_name_map = {}
-          errors = []
-
-          wait_for_output = lambda do
-            ready, = IO.select(out_fds + err_fds)
-            ready.each do |fd|
-              if fd.eof?
-                fd.close
-                out_fds.delete(fd)
-                err_fds.delete(fd)
-                next
-              end
-              line = fd.readline
-              puts "#{fd_to_name_map[fd]}: #{line}"
+            scopes = database_servers.to_h do |server|
+              [server, scope.merge(server.shards)]
             end
-          end
-
-          # only one process; don't bother forking
-          if scopes.length == 1 && parallel == 1
-            return with_each_shard(scopes.first.last.first, classes, exception: exception,
-                                   &block)
+          else
+            scopes = scope.group_by(&:database_server)
           end
 
           # clear connections prior to forking (no more queries will be executed in the parent,
           # and we want them gone so that we don't accidentally use them post-fork doing something
           # silly like dealloc'ing prepared statements)
-          ::ActiveRecord::Base.clear_all_connections!
-
-          scopes.each do |server, subscopes|
-            subscopes.each_with_index do |subscope, idx|
-              name = if subscopes.length > 1
-                       "#{server.id} #{idx + 1}"
-                     else
-                       server.id
-                     end
-
-              exception_pipe = IO.pipe
-              exception_pipes << exception_pipe
-              pid, io_in, io_out, io_err = Open4.pfork4(lambda do
-                Switchman.config[:on_fork_proc]&.call
-
-                # set a pretty name for the process title, up to 128 characters
-                # (we don't actually know the limit, depending on how the process
-                # was started)
-                # first, simplify the binary name by stripping directories,
-                # then truncate arguments as necessary
-                bin = File.basename($0) # Process.argv0 doesn't work on Ruby 2.5 (https://bugs.ruby-lang.org/issues/15887)
-                max_length = 128 - bin.length - name.length - 3
-                args = ARGV.join(' ')
-                args = args[0..max_length] if max_length >= 0
-                new_title = [bin, args, name].join(' ')
-                Process.setproctitle(new_title)
-
-                with_each_shard(subscope, classes, exception: exception, &block)
-                exception_pipe.last.close
-              rescue => e
-                begin
-                  dumped = Marshal.dump(e)
-                rescue
-                  # couldn't dump the exception; create a copy with just
-                  # the message and the backtrace
-                  e2 = e.class.new(e.message)
-                  e2.set_backtrace(e.backtrace)
-                  e2.instance_variable_set(:@active_shards, e.instance_variable_get(:@active_shards))
-                  dumped = Marshal.dump(e2)
-                end
-                exception_pipe.last.set_encoding(dumped.encoding)
-                exception_pipe.last.write(dumped)
-                exception_pipe.last.flush
-                exception_pipe.last.close
-                exit! 1
-              end)
-              exception_pipe.last.close
-              pids << pid
-              io_in.close # don't care about writing to stdin
-              out_fds << io_out
-              err_fds << io_err
-              pid_to_name_map[pid] = name
-              fd_to_name_map[io_out] = name
-              fd_to_name_map[io_err] = name
-
-              while max_procs && pids.count >= max_procs
-                while max_procs && out_fds.count >= max_procs
-                  # wait for output if we've hit the max_procs limit
-                  wait_for_output.call
-                end
-                # we've gotten all the output from one fd so wait for its child process to exit
-                found_pid, status = Process.wait2
-                pids.delete(found_pid)
-                errors << pid_to_name_map[found_pid] if status.exitstatus != 0
+          ::ActiveRecord::Base.connection_handler.clear_all_connections!(:all)
+
+          parent_process_name = sanitized_process_title
+          ret = ::Parallel.map(scopes, in_processes: (scopes.length > 1) ? parallel : 0) do |server, subscope|
+            name = server.id
+            last_description = name
+
+            begin
+              max_length = 128 - name.length - 3
+              short_parent_name = parent_process_name[0..max_length] if max_length >= 0
+              new_title = [short_parent_name, name].join(" ")
+              Process.setproctitle(new_title)
+              Switchman.config[:on_fork_proc]&.call
+              with_each_shard(subscope,
+                              classes,
+                              exception:,
+                              output: output || :decorated) do
+                last_description = Shard.current.description
+                Parallel::ResultWrapper.new(yield)
               end
+            rescue => e
+              logger.error e.full_message
+              Parallel::QuietExceptionWrapper.new(last_description, ::Parallel::ExceptionWrapper.new(e))
             end
-          end
-
-          wait_for_output.call while out_fds.any? || err_fds.any?
-          pids.each do |pid|
-            _, status = Process.waitpid2(pid)
-            errors << pid_to_name_map[pid] if status.exitstatus != 0
-          end
-
-          # check for an exception; we only re-raise the first one
-          exception_pipes.each do |exception_pipe|
-            serialized_exception = exception_pipe.first.read
-            next if serialized_exception.empty?
-
-            ex = Marshal.load(serialized_exception) # rubocop:disable Security/MarshalLoad
-            raise ex
-          ensure
-            exception_pipe.first.close
-          end
+          end.flatten
 
+          errors = ret.select { |val| val.is_a?(Parallel::QuietExceptionWrapper) }
           unless errors.empty?
-            raise ParallelShardExecError,
-                  "The following subprocesses did not exit cleanly: #{errors.sort.join(', ')}"
+            raise errors.first.exception if errors.length == 1
+
+            errors_desc = errors.map(&:name).sort.join(", ")
+            raise Errors::ParallelShardExecError,
+                  "The following database server(s) did not finish processing cleanly: #{errors_desc}",
+                  cause: errors.first.exception
           end
 
-          return
+          return ret.map(&:result)
         end
 
         classes ||= []
 
-        previous_shard = nil
         result = []
         ex = nil
+        old_stdout = $stdout
+        old_stderr = $stderr
         scope.each do |shard|
           # shard references a database server that isn't configured in this environment
           next unless shard.database_server
 
           shard.activate(*classes) do
+            if output
+              $stdout = Parallel::TransformingIO.new(output, $stdout)
+              $stderr = Parallel::TransformingIO.new(output, $stderr)
+            end
+
             result.concat Array.wrap(yield)
           rescue
             case exception
@@ -330,8 +284,10 @@ module Switchman
             else
               raise
             end
+          ensure
+            $stdout = old_stdout
+            $stderr = old_stderr
           end
-          previous_shard = shard
         end
         raise ex if ex
 
@@ -355,7 +311,7 @@ module Switchman
             else
               shard = partition_object.shard
             end
-          when Integer, /^\d+$/, /^(\d+)~(\d+)$/
+          when Integer, /\A\d+\Z/, /\A(\d+)~(\d+)\Z/
             local_id, shard = Shard.local_id_for(partition_object)
             local_id ||= partition_object
             object = local_id unless partition_proc
@@ -397,14 +353,14 @@ module Switchman
         case any_id
         when ::ActiveRecord::Base
           any_id.id
-        when /^(\d+)~(-?\d+)$/
+        when /\A(\d+)~(-?\d+)\Z/
           local_id = $2.to_i
           signed_id_operation(local_id) do |id|
             return nil if id > IDS_PER_SHARD
 
-            $1.to_i * IDS_PER_SHARD + id
+            ($1.to_i * IDS_PER_SHARD) + id
           end
-        when Integer, /^-?\d+$/
+        when Integer, /\A-?\d+\Z/
           any_id.to_i
         end
       end
@@ -480,32 +436,15 @@ module Switchman
         shard || source_shard || Shard.current
       end
 
-      # given the provided option, determines whether we need to (and whether
-      # it's possible) to determine a reasonable default.
-      def determine_max_procs(max_procs_input, parallel_input = 2)
-        max_procs = nil
-        if max_procs_input
-          max_procs = max_procs_input.to_i
-          max_procs = nil if max_procs.zero?
-        else
-          return 1 if parallel_input.nil? || parallel_input < 1
-
-          cpus = Environment.cpu_count
-          max_procs = cpus * parallel_input if cpus&.positive?
-        end
-
-        max_procs
-      end
-
       private
 
       def add_sharded_model(klass)
         @sharded_models = (sharded_models + [klass]).freeze
-        initialize_sharding
+        configure_connects_to
       end
 
-      def initialize_sharding
-        full_connects_to_hash = DatabaseServer.all.map { |db| [db.id.to_sym, db.connects_to_hash] }.to_h
+      def configure_connects_to
+        full_connects_to_hash = DatabaseServer.to_h { |db| [db.id.to_sym, db.connects_to_hash] }
         sharded_models.each do |klass|
           connects_to_hash = full_connects_to_hash.deep_dup
           if klass == UnshardedRecord
@@ -518,13 +457,16 @@ module Switchman
             connects_to_hash.each do |(db_name, role_hash)|
               role_hash.each_key do |role|
                 role_hash.delete(role) if klass.connection_handler.retrieve_connection_pool(
-                  klass.connection_specification_name, role: role, shard: db_name
+                  klass.connection_specification_name, role:, shard: db_name
                 )
               end
             end
           end
 
+          # this resets the default shard on rails 7.1+, but we want to preserve it
+          shard_was = klass.default_shard
           klass.connects_to shards: connects_to_hash
+          klass.default_shard = shard_was
         end
       end
 
@@ -558,8 +500,50 @@ module Switchman
         shard = nil unless shard.database_server
         shard
       end
+
+      # Determines the name of the current process, including arguments, but stripping
+      # any shebang from the invoked script, and any additional path info from the
+      # executable.
+      #
+      # @return [String]
+      def sanitized_process_title
+        # get the effective process name from `ps`; this will include any changes
+        # from Process.setproctitle _or_ assigning to $0.
+        parent_process_name = `ps -ocommand= -p#{Process.pid}`.strip
+        # Effective process titles may be shorter than the actual
+        # command; truncate our ARGV[0] so that they are comparable
+        # for the next step
+        argv0 = if parent_process_name.length < Process.argv0.length
+                  Process.argv0[0..parent_process_name.length]
+                else
+                  Process.argv0
+                end
+
+        # when running via a shebang, the `ps` output will include the shebang
+        # (i.e. it will be "ruby bin/rails c"); attempt to strip it off.
+        # Note that argv0 in this case will _only_ be `bin/rails` (no shebang,
+        # no arguments). We want to preserve the arguments we got from `ps`
+        if (index = parent_process_name.index(argv0))
+          parent_process_name.slice!(0...index)
+        end
+
+        # remove directories from the main executable to make more room
+        # for additional info
+        argv = parent_process_name.shellsplit
+        argv[0] = File.basename(argv[0])
+        argv.shelljoin
+      end
+
+      # @return [Array<String>] the list of database servers that are in the
+      #   config, but don't have any shards on them
+      def non_existent_database_servers
+        @non_existent_database_servers ||=
+          Shard.distinct.pluck(:database_server_id).compact - DatabaseServer.all.map(&:id)
+      end
     end
 
+    delegate :region, :in_region?, :in_current_region?, to: :database_server
+
     def name
       unless instance_variable_defined?(:@name)
         # protect against re-entrancy
@@ -588,7 +572,7 @@ module Switchman
     end
 
     def description
-      [database_server.id, name].compact.join(':')
+      [database_server.id, name].compact.join(":")
     end
 
     # Shards are always on the default shard
@@ -596,9 +580,17 @@ module Switchman
       Shard.default
     end
 
-    def activate(*classes, &block)
+    def original_id
+      id
+    end
+
+    def original_id_value
+      id
+    end
+
+    def activate(*classes, &)
       shards = hashify_classes(classes)
-      Shard.activate(shards, &block)
+      Shard.activate(shards, &)
     end
 
     # for use from console ONLY
@@ -618,48 +610,39 @@ module Switchman
     end
 
     def drop_database
-      raise('Cannot drop the database of the default shard') if default?
+      raise "Cannot drop the database of the default shard" if default?
       return unless read_attribute(:name)
 
       begin
-        adapter = database_server.config[:adapter]
-        sharding_config = Switchman.config || {}
-        drop_statement = sharding_config[adapter]&.[](:drop_statement)
-        drop_statement ||= sharding_config[:drop_statement]
-        if drop_statement
-          drop_statement = Array(drop_statement).dup.
-                           map { |statement| statement.gsub('%{name}', name) }
+        activate do
+          self.class.drop_database(name)
         end
+      rescue ::ActiveRecord::StatementInvalid => e
+        logger.error "Drop failed: #{e}"
+      end
+    end
 
-        case adapter
-        when 'mysql', 'mysql2'
-          activate do
-            ::GuardRail.activate(:deploy) do
-              drop_statement ||= "DROP DATABASE #{name}"
-              Array(drop_statement).each do |stmt|
-                ::ActiveRecord::Base.connection.execute(stmt)
-              end
-            end
-          end
-        when 'postgresql'
-          activate do
-            ::GuardRail.activate(:deploy) do
-              # Shut up, Postgres!
-              conn = ::ActiveRecord::Base.connection
-              old_proc = conn.raw_connection.set_notice_processor {}
-              begin
-                drop_statement ||= "DROP SCHEMA #{name} CASCADE"
-                Array(drop_statement).each do |stmt|
-                  ::ActiveRecord::Base.connection.execute(stmt)
-                end
-              ensure
-                conn.raw_connection.set_notice_processor(&old_proc) if old_proc
-              end
-            end
+    #
+    # Drops a specific database/schema from the currently active connection
+    #
+    def self.drop_database(name)
+      sharding_config = Switchman.config || {}
+      drop_statement = sharding_config["postgresql"]&.[](:drop_statement)
+      drop_statement ||= sharding_config[:drop_statement]
+      drop_statement = Array(drop_statement).map { |statement| statement.gsub("%{name}", name) } if drop_statement
+
+      ::GuardRail.activate(:deploy) do
+        # Shut up, Postgres!
+        conn = ::ActiveRecord::Base.connection
+        old_proc = conn.raw_connection.set_notice_processor {}
+        begin
+          drop_statement ||= "DROP SCHEMA #{name} CASCADE"
+          Array(drop_statement).each do |stmt|
+            ::ActiveRecord::Base.connection.execute(stmt)
           end
+        ensure
+          conn.raw_connection.set_notice_processor(&old_proc) if old_proc
         end
-      rescue
-        logger.info "Drop failed: #{$!}"
       end
     end
 
@@ -668,7 +651,7 @@ module Switchman
       return nil unless local_id
 
       self.class.signed_id_operation(local_id) do |abs_id|
-        abs_id + id * IDS_PER_SHARD
+        abs_id + (id * IDS_PER_SHARD)
       end
     end
 
@@ -678,7 +661,7 @@ module Switchman
     end
 
     def destroy
-      raise('Cannot destroy the default shard') if default?
+      raise("Cannot destroy the default shard") if default?
 
       super
     end
@@ -687,8 +670,8 @@ module Switchman
 
     def clear_cache
       Shard.default.activate do
-        Switchman.cache.delete(['shard', id].join('/'))
-        Switchman.cache.delete('default_shard') if default?
+        Switchman.cache.delete(["shard", id].join("/"))
+        Switchman.cache.delete("default_shard") if default?
       end
       self.class.clear_cache
     end