switchman 3.5.22 → 3.6.8

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: fa5da3b192752676ea8513c8fd05d72e347b10979b7b47ada9cdf008941fc2ac
-  data.tar.gz: 1da3c8eba9d967ca1a3edd550ecc7dd8bb1dea6193be3a752a65e95dc0beb54d
+  metadata.gz: 9c258da7abce6b4b34bb1dac83e352aeb4548776a34820969460c31b4c61674e
+  data.tar.gz: '0512961dad99c3041d461ed8e6e183a998d70cd4229712b7246099e8c408c139'
 SHA512:
-  metadata.gz: 7ba322eda93832f04595bb93f52153c61db3df1848404d80daf83e4b77b070f45c2d84cc2b6a5ba90c68cd5fbe26745f7ff2932b1a510ad15559f8234dab9bd2
-  data.tar.gz: 2453c5bccf84d20766340fe60512aec89547d27287832806964688c630f2af5d90070932c49e215f4a6ce209dfb701b1afa181ea1abe4fb8a009c03b55dea88e
+  metadata.gz: 494681d5982a2035e85f3058162a7864b1afc89df2c29ba0d95d99bc9408fabe23b4f762b706946152a3f5e983320bf0c2704c0cc66ad13367ae062c9d20843f
+  data.tar.gz: ce04f224bd153d4de41df5d2b8ec28b22c25e591eb6465ffcf6e8c1a5e831992981a6fc8c8e86130914ca5d91c6442c2b4335c76f374b61ae24bfc21ad838adc

data/lib/switchman/active_record/base.rb

@@ -22,13 +22,59 @@ module Switchman
           @integral_id
         end
 
-        %w[transaction insert_all upsert_all].each do |method|
+        def transaction(**)
+          if self != ::ActiveRecord::Base && current_scope
+            current_scope.activate do
+              db = Shard.current(connection_class_for_self).database_server
+              db.unguard { super }
+            end
+          else
+            db = Shard.current(connection_class_for_self).database_server
+            db.unguard { super }
+          end
+        end
+
+        # NOTE: `returning` values are _not_ transposed back to the current shard
+        %w[insert_all upsert_all].each do |method|
           class_eval <<-RUBY, __FILE__, __LINE__ + 1
-            def #{method}(*, **)
-              if self != ::ActiveRecord::Base && current_scope
-                current_scope.activate do
+            def #{method}(attributes, returning: nil, **)
+              scope = self != ::ActiveRecord::Base && current_scope
+              if (target_shard = scope&.primary_shard) == (current_shard = Shard.current(connection_class_for_self))
+                scope = nil
+              end
+              if scope
+                dupped = false
+                attributes.each_with_index do |hash, i|
+                  if dupped || hash.any? { |k, v| sharded_column?(k) }
+                    unless dupped
+                      attributes = attributes.dup
+                      dupped = true
+                    end
+                    attributes[i] = hash.to_h do |k, v|
+                      if sharded_column?(k)
+                        [k, Shard.relative_id_for(v, current_shard, target_shard)]
+                      else
+                        [k, v]
+                      end
+                    end
+                  end
+                end
+              end
+
+              if scope
+                scope.activate do
                   db = Shard.current(connection_class_for_self).database_server
-                  db.unguard { super }
+                  result = db.unguard { super }
+                  if result&.columns&.any? { |c| sharded_column?(c) }
+                    transposed_rows = result.rows.map do |row|
+                      row.map.with_index do |value, i|
+                        sharded_column?(result.columns[i]) ? Shard.relative_id_for(value, target_shard, current_shard) : value
+                      end
+                    end
+                    result = ::ActiveRecord::Result.new(result.columns, transposed_rows, result.column_types)
+                  end
+
+                  result
                 end
               else
                 db = Shard.current(connection_class_for_self).database_server
@@ -83,7 +129,7 @@ module Switchman
           config_or_env ||= if current_shard == ::Rails.env.to_sym && current_role == :primary
                               :primary
                             else
-                              "#{current_shard}/#{current_role}".to_sym
+                              :"#{current_shard}/#{current_role}"
                             end
 
           super(config_or_env)
@@ -153,8 +199,13 @@ module Switchman
 
       def self.prepended(klass)
         klass.singleton_class.prepend(ClassMethods)
-        klass.scope :non_shadow, ->(key = primary_key) { where("#{key}<=? AND #{key}>?", Shard::IDS_PER_SHARD, 0) }
-        klass.scope :shadow, ->(key = primary_key) { where("#{key}>?", Shard::IDS_PER_SHARD) }
+        klass.scope :non_shadow, lambda { |key = primary_key|
+                                   where(key => (QueryMethods::NonTransposingValue.new(0)..
+                                                 QueryMethods::NonTransposingValue.new(Shard::IDS_PER_SHARD)))
+                                 }
+        klass.scope :shadow, lambda { |key = primary_key|
+                               where(key => QueryMethods::NonTransposingValue.new(Shard::IDS_PER_SHARD)..)
+                             }
       end
 
       def _run_initialize_callbacks

data/lib/switchman/active_record/connection_pool.rb

@@ -64,7 +64,7 @@ module Switchman
       end
 
       def tls_key
-        "#{object_id}_shard".to_sym
+        :"#{object_id}_shard"
       end
     end
   end

data/lib/switchman/active_record/pending_migration_connection.rb

@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+
+module Switchman
+  module ActiveRecord
+    module PendingMigrationConnection
+      module ClassMethods
+        def current_role
+          ::ActiveRecord::Base.current_role
+        end
+
+        def current_switchman_shard
+          ::ActiveRecord::Base.current_switchman_shard
+        end
+      end
+    end
+  end
+end

data/lib/switchman/active_record/query_methods.rb

@@ -3,6 +3,19 @@
 module Switchman
   module ActiveRecord
     module QueryMethods
+      # Use this class to prevent a value from getting transposed across shards
+      class NonTransposingValue < SimpleDelegator
+        def class
+          __getobj__.class
+        end
+
+        def is_a?(other)
+          return true if other == NonTransposingValue
+
+          __getobj__.is_a?(other)
+        end
+      end
+
       # shard_value is one of:
       #   A shard
       #   An array or relation of shards
@@ -398,7 +411,9 @@ module Switchman
       end
 
       def transpose_predicate_value(value, current_shard, target_shard, attribute_type)
-        if value.is_a?(::ActiveRecord::StatementCache::Substitute)
+        if value.is_a?(NonTransposingValue)
+          value
+        elsif value.is_a?(::ActiveRecord::StatementCache::Substitute)
           value.sharded = true # mark for transposition later
           value.primary = true if attribute_type == :primary
           value

data/lib/switchman/active_record/relation.rb

@@ -44,8 +44,14 @@ module Switchman
         primary_shard.activate(klass.connection_class_for_self) { super }
       end
 
-      def explain
-        activate { |relation| relation.call_super(:explain, Relation) }
+      if ::Rails.version > "7.1.2"
+        def transaction(...)
+          primary_shard.activate(klass.connection_class_for_self) { super }
+        end
+      end
+
+      def explain(*args)
+        activate { |relation| relation.call_super(:explain, Relation, *args) }
       end
 
       def load(&block)
@@ -58,10 +64,9 @@ module Switchman
       end
 
       %I[update_all delete_all].each do |method|
-        arg_params = (RUBY_VERSION <= "2.8") ? "*args" : "*args, **kwargs"
         class_eval <<-RUBY, __FILE__, __LINE__ + 1
-          def #{method}(#{arg_params})
-            result = self.activate(unordered: true) { |relation| relation.call_super(#{method.inspect}, Relation, #{arg_params}) }
+          def #{method}(*args, **kwargs)
+            result = self.activate(unordered: true) { |relation| relation.call_super(#{method.inspect}, Relation, *args, **kwargs) }
             result = result.sum if result.is_a?(Array)
             result
           end

data/lib/switchman/active_support/cache.rb

@@ -22,12 +22,14 @@ module Switchman
 
         def lookup_store(*store_options)
           store = super
-          # can't use defined?, because it's a _ruby_ autoloaded constant,
-          # so just checking that will cause it to get required
-          if store.instance_of?(ActiveSupport::Cache::RedisCacheStore) &&
-             !::ActiveSupport::Cache::RedisCacheStore <= RedisCacheStore
+          # must use the string name, otherwise it will try to auto-load the constant
+          # and we don't want to require redis in this file (since it's not a hard dependency)
+          # rubocop:disable Style/ClassEqualityComparison
+          if store.class.name == "ActiveSupport::Cache::RedisCacheStore" &&
+             !(::ActiveSupport::Cache::RedisCacheStore <= RedisCacheStore)
             ::ActiveSupport::Cache::RedisCacheStore.prepend(RedisCacheStore)
           end
+          # rubocop:enable Style/ClassEqualityComparison
           store.options[:namespace] ||= -> { Shard.current.default? ? nil : "shard_#{Shard.current.id}" }
           store
         end
@@ -36,7 +38,7 @@ module Switchman
       module RedisCacheStore
         def clear(namespace: nil, **)
           # RedisCacheStore tries to be smart and only clear the cache under your namespace, if you have one set
-          # unfortunately, it uses the keys command, which is extraordinarily inefficient in a large redis instance
+          # unfortunately, it doesn't work using redis clustering because of the way redis keys are distributed
           # fortunately, we can assume we control the entire instance, because we set up the namespacing, so just
           # always unset it temporarily for clear calls
           namespace = nil # rubocop:disable Lint/ShadowedArgument

data/lib/switchman/call_super.rb

@@ -12,14 +12,8 @@ module Switchman
       method.super_method
     end
 
-    if RUBY_VERSION <= "2.8"
-      def call_super(method, above_module, *args, &block)
-        super_method_above(method, above_module).call(*args, &block)
-      end
-    else
-      def call_super(method, above_module, *args, **kwargs, &block)
-        super_method_above(method, above_module).call(*args, **kwargs, &block)
-      end
+    def call_super(method, above_module, ...)
+      super_method_above(method, above_module).call(...)
     end
   end
 end

data/lib/switchman/engine.rb

@@ -78,6 +78,11 @@ module Switchman
         ::ActiveRecord::MigrationContext.prepend(ActiveRecord::MigrationContext)
         ::ActiveRecord::Migrator.prepend(ActiveRecord::Migrator)
 
+        if ::Rails.version > "7.1.3"
+          ::ActiveRecord::PendingMigrationConnection.singleton_class
+                                                    .include(ActiveRecord::PendingMigrationConnection::ClassMethods)
+        end
+
         ::ActiveRecord::Reflection::AbstractReflection.include(ActiveRecord::Reflection::AbstractReflection)
         ::ActiveRecord::Reflection::AssociationReflection.prepend(ActiveRecord::Reflection::AssociationScopeCache)
         ::ActiveRecord::Reflection::ThroughReflection.prepend(ActiveRecord::Reflection::AssociationScopeCache)

data/lib/switchman/guard_rail/relation.rb

@@ -13,9 +13,8 @@ module Switchman
       end
 
       %w[update_all delete_all].each do |method|
-        arg_params = (RUBY_VERSION <= "2.8") ? "*args" : "*args, **kwargs"
         class_eval <<-RUBY, __FILE__, __LINE__ + 1
-          def #{method}(#{arg_params})
+          def #{method}(*args, **kwargs)
             db = Shard.current(connection_class_for_self).database_server
             db.unguard { super }
           end

data/lib/switchman/parallel.rb

@@ -50,16 +50,16 @@ module Switchman
       end
     end
 
-    class PrefixingIO
+    class TransformingIO
       delegate_missing_to :@original_io
 
-      def initialize(prefix, original_io)
-        @prefix = prefix
+      def initialize(transformer, original_io)
+        @transformer = transformer
         @original_io = original_io
       end
 
       def puts(*args)
-        args.flatten.each { |arg| @original_io.puts "#{@prefix}: #{arg}" }
+        args.flatten.each { |arg| @original_io.puts @transformer.call(arg) }
       end
     end
   end

data/lib/switchman/r_spec_helper.rb

@@ -126,6 +126,7 @@ module Switchman
             ds.shards.delete_all unless use_transactional_tests
             ds.destroy
           end
+          ds.remove_instance_variable(:@primary_shard_id) if ds.instance_variable_defined?(:@primary_shard_id)
         end
       end
 

data/lib/switchman/shard.rb

@@ -165,7 +165,7 @@ module Switchman
       #                forking.
       #    exception: - :ignore, :raise, :defer (wait until the end and raise the first
       #                error), or a proc
-      #    output: - :simple, :decorated (with database_server_id:shard_name)
+      #    output: - :simple, :decorated (with database_server_id:shard_name), custom lambda transformer
       def with_each_shard(*args, parallel: false, exception: :raise, output: :simple)
         raise ArgumentError, "wrong number of arguments (#{args.length} for 0...2)" if args.length > 2
 
@@ -181,6 +181,13 @@ module Switchman
           scope, classes = args
         end
 
+        output = if output == :decorated
+                   ->(arg) { "#{Shard.current.description}: #{arg}" }
+                 elsif output == :simple
+                   nil
+                 else
+                   output
+                 end
         parallel = [Environment.cpu_count || 2, 2].min if parallel == true
         parallel = 0 if parallel == false || parallel.nil?
 
@@ -224,7 +231,10 @@ module Switchman
               new_title = [short_parent_name, name].join(" ")
               Process.setproctitle(new_title)
               Switchman.config[:on_fork_proc]&.call
-              with_each_shard(subscope, classes, exception: exception, output: :decorated) do
+              with_each_shard(subscope,
+                              classes,
+                              exception: exception,
+                              output: output || :decorated) do
                 last_description = Shard.current.description
                 Parallel::ResultWrapper.new(yield)
               end
@@ -258,9 +268,9 @@ module Switchman
           next unless shard.database_server
 
           shard.activate(*classes) do
-            if output == :decorated
-              $stdout = Parallel::PrefixingIO.new(shard.description, $stdout)
-              $stderr = Parallel::PrefixingIO.new(shard.description, $stderr)
+            if output
+              $stdout = Parallel::TransformingIO.new(output, $stdout)
+              $stderr = Parallel::TransformingIO.new(output, $stderr)
             end
 
             result.concat Array.wrap(yield)

data/lib/switchman/sharded_instrumenter.rb

@@ -3,7 +3,7 @@
 module Switchman
   class ShardedInstrumenter < ::SimpleDelegator
     def initialize(instrumenter, shard_host)
-      super instrumenter
+      super(instrumenter)
       @shard_host = shard_host
     end
 
@@ -23,7 +23,7 @@ module Switchman
                end
         }
       end
-      super name, payload
+      super(name, payload)
     end
   end
 end

data/lib/switchman/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Switchman
-  VERSION = "3.5.22"
+  VERSION = "3.6.8"
 end

data/lib/tasks/switchman.rake

@@ -71,12 +71,13 @@ module Switchman
     end
 
     def self.options
-      { parallel: ENV["PARALLEL"].to_i }
+      { exception: (ENV["FAIL_FAST"] == "0") ? :defer : :raise, parallel: ENV["PARALLEL"].to_i }
     end
 
     # classes - an array or proc, to activate as the current shard during the
     # task.
     def self.shardify_task(task_name, classes: [::ActiveRecord::Base])
+      log_format = ENV.fetch("LOG_FORMAT", nil)
       old_task = ::Rake::Task[task_name]
       old_actions = old_task.actions.dup
       old_task.actions.clear
@@ -90,14 +91,56 @@ module Switchman
         ::GuardRail.activate(:deploy) do
           Shard.default.database_server.unguard do
             classes = classes.call if classes.respond_to?(:call)
-            Shard.with_each_shard(scope, classes, **options) do
+
+            # We don't want the shard status messages to be wrapped using a custom log transfomer
+            original_stderr = $stderr
+            original_stdout = $stdout
+            output = if log_format == "json"
+                       lambda { |msg|
+                         JSON.dump(shard: Shard.current.id,
+                                   database_server: Shard.current.database_server.id,
+                                   type: "log",
+                                   message: msg)
+                       }
+                     else
+                       nil
+                     end
+            Shard.with_each_shard(scope, classes, output: output, **options) do
               shard = Shard.current
-              puts "#{shard.id}: #{shard.description}"
+
+              if log_format == "json"
+                original_stdout.puts JSON.dump(
+                  shard: shard.id,
+                  database_server: shard.database_server.id,
+                  type: "started"
+                )
+              else
+                original_stdout.puts "#{shard.id}: #{shard.description}"
+              end
 
               shard.database_server.unguard do
                 old_actions.each { |action| action.call(*task_args) }
               end
+
+              if log_format == "json"
+                original_stdout.puts JSON.dump(
+                  shard: shard.id,
+                  database_server: shard.database_server.id,
+                  type: "completed"
+                )
+              end
               nil
+            rescue => e
+              if log_format == "json"
+                original_stderr.puts JSON.dump(
+                  shard: shard.id,
+                  database_server: shard.database_server.id,
+                  type: "failed",
+                  message: e.full_message
+                )
+              end
+
+              raise
             end
           rescue => e
             if options[:parallel] != 0

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: switchman
 version: !ruby/object:Gem::Version
-  version: 3.5.22
+  version: 3.6.8
 platform: ruby
 authors:
 - Cody Cutrer
@@ -10,7 +10,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2024-02-26 00:00:00.000000000 Z
+date: 2025-01-27 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activerecord
@@ -249,6 +249,7 @@ files:
 - lib/switchman/active_record/log_subscriber.rb
 - lib/switchman/active_record/migration.rb
 - lib/switchman/active_record/model_schema.rb
+- lib/switchman/active_record/pending_migration_connection.rb
 - lib/switchman/active_record/persistence.rb
 - lib/switchman/active_record/postgresql_adapter.rb
 - lib/switchman/active_record/predicate_builder.rb
@@ -297,14 +298,14 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: '2.7'
+      version: '3.0'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.1.6
+rubygems_version: 3.2.33
 signing_key: 
 specification_version: 4
 summary: Rails sharding magic