karafka 2.5.9 → 2.6.0.beta1

data/lib/karafka/processing/strategy_selector.rb

@@ -1,48 +1,10 @@
 # frozen_string_literal: true
 
+# Backwards-compatible alias kept for external code that references the old, un-namespaced
+# constant. Will be removed in Karafka 3.0.
 module Karafka
   module Processing
-    # Selector of appropriate processing strategy matching topic combinations
-    class StrategySelector
-      attr_reader :strategies
-
-      # Features we support in the OSS offering.
-      SUPPORTED_FEATURES = %i[
-        active_job
-        manual_offset_management
-        dead_letter_queue
-      ].freeze
-
-      # Initializes the strategy selector and preloads all strategies
-      def initialize
-        # We load them once for performance reasons not to do too many lookups
-        @strategies = find_all
-      end
-
-      # @param topic [Karafka::Routing::Topic] topic with settings based on which we find strategy
-      # @return [Module] module with proper strategy
-      def find(topic)
-        feature_set = SUPPORTED_FEATURES.map do |feature_name|
-          topic.public_send("#{feature_name}?") ? feature_name : nil
-        end
-
-        feature_set.compact!
-
-        @strategies.find do |strategy|
-          strategy::FEATURES.sort == feature_set.sort
-        end || raise(Errors::StrategyNotFoundError, topic.name)
-      end
-
-      private
-
-      # @return [Array<Module>] available strategies
-      def find_all
-        Strategies
-          .constants
-          .delete_if { |k| k == :Base }
-          .map { |k| Strategies.const_get(k) }
-          .uniq
-      end
-    end
+    # @see ConsumerGroups::StrategySelector
+    StrategySelector = ConsumerGroups::StrategySelector
   end
 end

data/lib/karafka/processing/worker.rb

@@ -27,17 +27,18 @@ module Karafka
       attr_reader :id
 
       # @param jobs_queue [JobsQueue]
+      # @param pool [WorkersPool] pool this worker belongs to (for deregistration on shutdown)
       # @return [Worker]
-      def initialize(jobs_queue)
+      def initialize(jobs_queue, pool)
         @id = SecureRandom.hex(6)
         @jobs_queue = jobs_queue
+        @pool = pool
         @non_wrapped_flow = worker_job_call_wrapper == false
       end
 
       private
 
-      # Runs processing of jobs in a loop
-      # Stops when queue is closed.
+      # Runs processing of jobs in a loop Stops when queue is closed.
       def call
         loop { break unless process }
       end
@@ -80,6 +81,9 @@ module Karafka
             end
           end
         else
+          # nil means either queue closed (full shutdown) or pool downscaling.
+          # Either way, deregister from pool so it reflects the actual worker count.
+          @pool.deregister(self)
           false
         end
       # We signal critical exceptions, notify and do not allow worker to fail
@@ -95,7 +99,7 @@ module Karafka
           type: "worker.process.error"
         )
       ensure
-        # job can be nil when the queue is being closed
+        # job can be nil when the queue is being closed or during pool downscaling
         if job
           @jobs_queue.complete(job)
           job.finish!

data/lib/karafka/processing/workers_pool.rb

@@ -0,0 +1,158 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Processing
+    # Dynamic thread pool that manages worker threads.
+    # Supports scaling at runtime via {#scale}.
+    #
+    # All public methods that read or mutate `@workers` are synchronized via `@mutex`.
+    # `@size` is always updated under `@mutex` but can be read without locking for performance
+    # (integer assignment is atomic in MRI).
+    class WorkersPool
+      include Helpers::ConfigImporter.new(
+        concurrency: %i[concurrency],
+        worker_thread_priority: %i[worker_thread_priority],
+        monitor: %i[monitor]
+      )
+
+      # @return [Integer] current number of workers registered in the pool.
+      #   Reflects the actual thread count, not a target. After a downscale request this value
+      #   converges towards the target as workers pick up nil sentinels and deregister.
+      #   Updated atomically under mutex, safe to read without locking.
+      attr_reader :size
+
+      # Jobs queue reference, set by the Runner after both pool and queue are created.
+      # Must be assigned before calling {#scale}.
+      attr_writer :jobs_queue
+
+      # Initializes an empty pool with zero workers.
+      # Workers are not started until {#scale} is called, allowing the pool to be created early
+      # (e.g. in Server.run) before the jobs queue exists.
+      #
+      # @return [WorkersPool]
+      def initialize
+        @jobs_queue = nil
+        @workers = []
+        @size = 0
+        @mutex = Mutex.new
+        # Monotonically increasing index for naming worker threads. Indices are never reused
+        # after a worker exits, so thread names remain unique across the lifetime of the process
+        # and make it easy to correlate log entries with specific worker generations.
+        @next_index = 0
+      end
+
+      # Scale pool towards `target` workers (minimum 1).
+      #
+      # **Scaling up** is synchronous -- new worker threads are spawned and registered before this
+      # method returns. {#size} reflects the new count immediately.
+      #
+      # **Scaling down** is asynchronous -- nil sentinels are enqueued and workers exit when they
+      # pick one up. {#size} decreases gradually as workers deregister themselves. Callers that
+      # need to know when downsizing is complete should poll {#size} or listen for the
+      # `worker.scaling.down` instrumentation event (whose `:to` payload reports the *target*,
+      # not the current count).
+      #
+      # The entire read-decide-act cycle is synchronized to prevent stale reads.
+      # Instrumentation runs outside the mutex to avoid holding the lock during user callbacks.
+      #
+      # @param target [Integer] desired number of workers
+      def scale(target)
+        raise(Karafka::Errors::BaseError, "jobs_queue must be set before scaling") unless @jobs_queue
+
+        target = [target, 1].max
+        event = nil
+
+        @mutex.synchronize do
+          current = @workers.size
+          delta = target - current
+
+          if delta.positive?
+            event = grow(delta)
+          elsif delta.negative?
+            event = shrink(delta.abs)
+          end
+        end
+
+        return unless event
+
+        monitor.instrument(*event)
+      end
+
+      # @return [Boolean] true if all workers have stopped
+      def stopped?
+        snapshot.none?(&:alive?)
+      end
+
+      # @return [Array<Worker>] workers that are still alive
+      def alive
+        snapshot.select(&:alive?)
+      end
+
+      # Forcefully terminate all worker threads.
+      def terminate
+        snapshot.each(&:terminate)
+      end
+
+      # Wait for all current workers to finish.
+      def join
+        snapshot.each(&:join)
+      end
+
+      # Called by a worker when it exits (queue closed or pool downscaling).
+      # Thread-safe -- worker threads call this from their own thread.
+      #
+      # @param worker [Worker] worker to remove from the pool
+      def deregister(worker)
+        @mutex.synchronize do
+          @workers.delete(worker)
+          @size = @workers.size
+        end
+      end
+
+      private
+
+      # @return [Array<Worker>] snapshot of workers taken under mutex
+      def snapshot
+        @mutex.synchronize { @workers.dup }
+      end
+
+      # Add `count` workers and start their threads immediately.
+      # Must be called under `@mutex` (from {#scale}) or during construction (no contention).
+      #
+      # @param count [Integer] number of workers to add
+      # @return [Array] instrumentation event args to be emitted outside the mutex
+      def grow(count)
+        from = @workers.size
+
+        count.times do
+          worker = Worker.new(@jobs_queue, self)
+          @workers << worker
+          worker.async_call("karafka.worker##{@next_index}", worker_thread_priority)
+          @next_index += 1
+        end
+
+        @size = @workers.size
+
+        ["worker.scaling.up", { workers_pool: self, from: from, to: @size }]
+      end
+
+      # Push nil into the queue to signal workers to exit.
+      # Whichever workers pick them up will deregister and stop.
+      # Must be called under `@mutex` (from {#scale}).
+      #
+      # @param count [Integer] number of workers to remove
+      # @return [Array, nil] instrumentation event args or nil if no-op
+      # @note Never shrinks below 1 worker.
+      def shrink(count)
+        effective = [count, @workers.size - 1].min
+        return if effective <= 0
+
+        from = @workers.size
+        effective.times { @jobs_queue << nil }
+        to = from - effective
+
+        ["worker.scaling.down", { workers_pool: self, from: from, to: to }]
+      end
+    end
+  end
+end

data/lib/karafka/routing/builder.rb

@@ -56,23 +56,23 @@ module Karafka
             scope: %w[routes]
           )
 
-          each do |consumer_group|
-            # Validate consumer group settings
+          each do |group|
+            # Validate group settings
             Contracts::ConsumerGroup.new.validate!(
-              consumer_group.to_h,
-              scope: ["routes", consumer_group.name]
+              group.to_h,
+              scope: ["routes", group.name]
             )
 
             # and then its topics settings
-            consumer_group.topics.each do |topic|
+            group.topics.each do |topic|
               Contracts::Topic.new.validate!(
                 topic.to_h,
-                scope: ["routes", consumer_group.name, topic.name]
+                scope: ["routes", group.name, topic.name]
               )
             end
 
             # Initialize subscription groups after all the routing is done
-            consumer_group.subscription_groups
+            group.subscription_groups
           end
         end
       end
@@ -125,13 +125,13 @@ module Karafka
       # Builds and saves given consumer group
       # @param group_id [String, Symbol] name for consumer group
       def consumer_group(group_id, &)
-        consumer_group = find { |cg| cg.name == group_id.to_s }
+        group = find { |existing| existing.name == group_id.to_s }
 
-        if consumer_group
-          Proxy.new(consumer_group, &).target
+        if group
+          Proxy.new(group, &).target
         else
-          consumer_group = ConsumerGroup.new(group_id.to_s)
-          self << Proxy.new(consumer_group, &).target
+          group = ConsumerGroup.new(group_id.to_s)
+          self << Proxy.new(group, &).target
         end
       end
 

data/lib/karafka/routing/contracts/routing.rb

@@ -26,11 +26,10 @@ module Karafka
           # All topics including the DLQ topics names that are marked as active
           topics = Set.new
 
-          data.each do |consumer_group|
-            consumer_group[:topics].each do |topic|
+          data.each do |group|
+            group[:topics].each do |topic|
               pat = topic[:patterns]
-              # Ignore pattern topics because they won't exist and should not be declarative
-              # managed
+              # Ignore pattern topics because they won't exist and should not be declarative managed
               topics << topic[:name] if !pat || !pat[:active]
 
               dlq = topic[:dead_letter_queue]

data/lib/karafka/routing/features/base/expander.rb

@@ -36,20 +36,20 @@ module Karafka
               define_method :draw do |&block|
                 result = super(&block)
 
-                each do |consumer_group|
+                each do |group|
                   if scope::Contracts.const_defined?("ConsumerGroup", false)
                     scope::Contracts::ConsumerGroup.new.validate!(
-                      consumer_group.to_h,
-                      scope: ["routes", consumer_group.name]
+                      group.to_h,
+                      scope: ["routes", group.name]
                     )
                   end
 
                   next unless scope::Contracts.const_defined?("Topic", false)
 
-                  consumer_group.topics.each do |topic|
+                  group.topics.each do |topic|
                     scope::Contracts::Topic.new.validate!(
                       topic.to_h,
-                      scope: ["routes", consumer_group.name, topic.name]
+                      scope: ["routes", group.name, topic.name]
                     )
                   end
                 end

data/lib/karafka/routing/features/consumer_groups/active_job/builder.rb

@@ -0,0 +1,35 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Routing
+    module Features
+      module ConsumerGroups
+        class ActiveJob < Base
+          # Routing extensions for ActiveJob
+          module Builder
+            # This method simplifies routes definition for ActiveJob topics / queues by
+            # auto-injecting the consumer class
+            #
+            # @param name [String, Symbol] name of the topic where ActiveJob jobs should go
+            # @param block [Proc] block that we can use for some extra configuration
+            def active_job_topic(name, &block)
+              topic(name) do
+                consumer App.config.internal.active_job.consumer_class
+                active_job true
+
+                # This is handled by our custom ActiveJob consumer
+                # Without this, default behaviour would cause messages to skip upon shutdown as the
+                # offset would be committed for the last message
+                manual_offset_management true
+
+                next unless block
+
+                instance_eval(&block)
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/routing/features/consumer_groups/active_job/config.rb

@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Routing
+    module Features
+      module ConsumerGroups
+        class ActiveJob < Base
+          # Config for ActiveJob usage
+          Config = Struct.new(
+            :active,
+            keyword_init: true
+          ) { alias_method :active?, :active }
+        end
+      end
+    end
+  end
+end

data/lib/karafka/routing/features/consumer_groups/active_job/contracts/topic.rb

@@ -0,0 +1,44 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Routing
+    module Features
+      module ConsumerGroups
+        class ActiveJob < Base
+          # This feature validation contracts
+          module Contracts
+            # Rules around using ActiveJob routing - basically you need to have ActiveJob available
+            # in order to be able to use active job routing
+            class Topic < Karafka::Contracts::Base
+              configure do |config|
+                config.error_messages = YAML.safe_load_file(
+                  File.join(Karafka.gem_root, "config", "locales", "errors.yml")
+                ).fetch("en").fetch("validations").fetch("routing").fetch("topic")
+              end
+
+              virtual do |data, errors|
+                next unless errors.empty?
+                next unless data[:active_job][:active]
+                # One should not define active job jobs without ActiveJob being available for usage
+                next if Object.const_defined?("ActiveJob::Base")
+
+                [[%i[consumer], :active_job_missing]]
+              end
+
+              # ActiveJob needs to always run with manual offset management
+              # Automatic offset management cannot work with ActiveJob. Otherwise we could mark as
+              # consumed jobs that did not run because of shutdown.
+              virtual do |data, errors|
+                next unless errors.empty?
+                next unless data[:active_job][:active]
+                next if data[:manual_offset_management][:active]
+
+                [[%i[manual_offset_management], :must_be_enabled]]
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/routing/features/consumer_groups/active_job/proxy.rb

@@ -0,0 +1,16 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Routing
+    module Features
+      module ConsumerGroups
+        class ActiveJob < Base
+          # Routing proxy extensions for ActiveJob
+          module Proxy
+            include Builder
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/routing/features/consumer_groups/active_job/topic.rb

@@ -0,0 +1,50 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Routing
+    module Features
+      module ConsumerGroups
+        class ActiveJob < Base
+          # Topic extensions to be able to check if given topic is ActiveJob topic
+          #
+          # @note ActiveJob topics do not have per-topic deserializer configuration. The deserializer
+          #   is configured globally via `config.internal.active_job.deserializer` because Rails
+          #   serializes jobs before dispatching them, requiring a consistent serialization format
+          #   across all ActiveJob topics. If you need custom serialization (e.g., Avro, Protobuf),
+          #   configure it once at the application level rather than per-topic.
+          module Topic
+            # This method sets up the extra instance variable to nil before calling
+            # the parent class initializer. The explicit initialization
+            # to nil is included as an optimization for Ruby's object shapes system,
+            # which improves memory layout and access performance.
+            def initialize(...)
+              @active_job = nil
+              super
+            end
+
+            # @param active [Boolean] should this topic be considered one working with ActiveJob
+            #
+            # @note Since this feature supports only one setting (active), we can use the old API
+            # where the boolean would be an argument
+            def active_job(active = false)
+              @active_job ||= Config.new(active: active)
+            end
+
+            # @return [Boolean] is this an ActiveJob topic
+            def active_job?
+              active_job.active?
+            end
+
+            # @return [Hash] topic with all its native configuration options plus active job
+            #   namespace settings
+            def to_h
+              super.merge(
+                active_job: active_job.to_h
+              ).freeze
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/routing/features/consumer_groups/active_job.rb

@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Routing
+    module Features
+      module ConsumerGroups
+        # Active-Job related components
+        # @note We can load it always, despite someone not using ActiveJob as it just adds a method
+        #   to the routing, without actually breaking anything.
+        class ActiveJob < Base
+        end
+      end
+    end
+  end
+end

data/lib/karafka/routing/features/consumer_groups/dead_letter_queue/config.rb

@@ -0,0 +1,39 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Routing
+    module Features
+      module ConsumerGroups
+        class DeadLetterQueue < Base
+          # Config for dead letter queue feature
+          Config = Struct.new(
+            :active,
+            # We add skip variants but in regular we support only `:one`
+            :max_retries,
+            # To what topic the skipped messages should be moved
+            :topic,
+            # Should retries be handled collectively on a batch or independently per message
+            :independent,
+            # Move to DLQ and mark as consumed in transactional mode (if applicable)
+            :transactional,
+            # Strategy to apply (if strategies supported)
+            :strategy,
+            # Should we use `#produce_sync` or `#produce_async`
+            :dispatch_method,
+            # Should we use `#mark_as_consumed` or `#mark_as_consumed!` (in flows that mark)
+            :marking_method,
+            # Should we mark as consumed after dispatch or not. True for most cases, except MOM where
+            # it is on user to decide (false by default)
+            :mark_after_dispatch,
+            # Initialize with kwargs
+            keyword_init: true
+          ) do
+            alias_method :active?, :active
+            alias_method :independent?, :independent
+            alias_method :transactional?, :transactional
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/routing/features/consumer_groups/dead_letter_queue/contracts/topic.rb

@@ -0,0 +1,58 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Routing
+    module Features
+      module ConsumerGroups
+        class DeadLetterQueue < Base
+          # This feature validation contracts
+          module Contracts
+            # Rules around dead letter queue settings
+            class Topic < Karafka::Contracts::Base
+              configure do |config|
+                config.error_messages = YAML.safe_load_file(
+                  File.join(Karafka.gem_root, "config", "locales", "errors.yml")
+                ).fetch("en").fetch("validations").fetch("routing").fetch("topic")
+              end
+
+              nested :dead_letter_queue do
+                required(:active) { |val| [true, false].include?(val) }
+                required(:independent) { |val| [true, false].include?(val) }
+                required(:max_retries) { |val| val.is_a?(Integer) && val >= 0 }
+                required(:transactional) { |val| [true, false].include?(val) }
+                required(:mark_after_dispatch) { |val| [true, false, nil].include?(val) }
+
+                required(:dispatch_method) do |val|
+                  %i[produce_async produce_sync].include?(val)
+                end
+
+                required(:marking_method) do |val|
+                  %i[mark_as_consumed mark_as_consumed!].include?(val)
+                end
+              end
+
+              # Validate topic name only if dlq is active
+              virtual do |data, errors|
+                next unless errors.empty?
+
+                dead_letter_queue = data[:dead_letter_queue]
+
+                next unless dead_letter_queue[:active]
+
+                topic = dead_letter_queue[:topic]
+                topic_regexp = Karafka::Contracts::TOPIC_REGEXP
+
+                # When topic is set to false, it means we just want to skip dispatch on DLQ
+                next if topic == false
+                next if topic.is_a?(String) && topic_regexp.match?(topic)
+                next if topic == :strategy
+
+                [[%i[dead_letter_queue topic], :format]]
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end