karafka 2.5.9 → 2.6.0.beta1

data/lib/karafka/routing/topic.rb

@@ -3,12 +3,22 @@
 module Karafka
   module Routing
     # Topic stores all the details on how we should interact with Kafka given topic.
-    # It belongs to a consumer group as from 0.6 all the topics can work in the same consumer group
+    # It belongs to a group as from 0.6 all the topics can work in the same group
     # It is a part of Karafka's DSL.
+    #
+    # @note `#group` is the polymorphic reference to the owning group. Today this is always a
+    #   {ConsumerGroup}, but the accessor is named generically in preparation for additional group
+    #   types (e.g. KIP-932 share groups). `#consumer_group` is kept as an alias for backwards
+    #   compatibility.
     class Topic
-      attr_reader :id, :name, :consumer_group
+      attr_reader :id, :name, :group
       attr_writer :consumer
 
+      # Backwards compatible alias for `#group`. Kept purely for compatibility — this is an
+      # unconditional alias and performs no type validation, so callers should prefer `#group`
+      # once additional group types (e.g. KIP-932 share groups) land.
+      alias_method :consumer_group, :group
+
       attr_accessor :subscription_group_details
 
       # Full subscription group reference can be built only when we have knowledge about the
@@ -30,16 +40,17 @@ module Karafka
       private_constant :INHERITABLE_ATTRIBUTES
 
       # @param name [String, Symbol] name of a topic on which we want to listen
-      # @param consumer_group [Karafka::Routing::ConsumerGroup] owning consumer group of this topic
-      def initialize(name, consumer_group)
+      # @param group [Karafka::Routing::ConsumerGroup] owning group of this topic. Polymorphic
+      #   placeholder for future group types (e.g. share groups); today always a ConsumerGroup.
+      def initialize(name, group)
         @name = name.to_s
-        @consumer_group = consumer_group
+        @group = group
         @attributes = {}
         @active = true
-        # @note We use identifier related to the consumer group that owns a topic, because from
-        #   Karafka 0.6 we can handle multiple Kafka instances with the same process and we can
-        #   have same topic name across multiple consumer groups
-        @id = "#{consumer_group.id}_#{@name}"
+        # @note We use identifier related to the group that owns a topic, because from Karafka 0.6
+        #   we can handle multiple Kafka instances with the same process and we can have same
+        #   topic name across multiple groups
+        @id = "#{group.id}_#{@name}"
         @consumer = nil
         @active_assigned = false
         @subscription_group_details = nil
@@ -133,7 +144,7 @@ module Karafka
       end
 
       # @return [Hash] hash with all the topic attributes
-      # @note This is being used when we validate the consumer_group and its topics
+      # @note This is being used when we validate the group and its topics
       def to_h
         map = INHERITABLE_ATTRIBUTES.map do |attribute|
           [attribute, public_send(attribute)]
@@ -144,7 +155,10 @@ module Karafka
           name: name,
           active: active?,
           consumer: consumer,
-          consumer_group_id: consumer_group.id,
+          group_id: group.id,
+          # Kept as a reference alongside `group_id` for backwards compatibility. Will be removed
+          # in Karafka 3.0.
+          consumer_group_id: group.id,
           subscription_group_details: subscription_group_details
         ).freeze
       end

data/lib/karafka/runner.rb

@@ -4,20 +4,20 @@ module Karafka
   # Class used to run the Karafka listeners in separate threads
   class Runner
     include Helpers::ConfigImporter.new(
-      worker_thread_priority: %i[worker_thread_priority],
       manager: %i[internal connection manager],
-      conductor: %i[internal connection conductor],
-      jobs_queue_class: %i[internal processing jobs_queue_class]
+      conductor: %i[internal connection conductor]
     )
 
     # Starts listening on all the listeners asynchronously and handles the jobs queue closing
     # after listeners are done with their work.
     def call
-      # Despite possibility of having several independent listeners, we aim to have one queue for
-      # jobs across and one workers poll for that
-      jobs_queue = jobs_queue_class.new
+      jobs_queue = Karafka::Server.jobs_queue
+      workers = Karafka::Server.workers
+
+      # Wire up the circular dependency between pool and queue
+      workers.jobs_queue = jobs_queue
+      jobs_queue.pool = workers
 
-      workers = Processing::WorkersBatch.new(jobs_queue)
       listeners = Connection::ListenersBatch.new(jobs_queue)
 
       # We mark it prior to delegating to the manager as manager will have to start at least one
@@ -27,17 +27,12 @@ module Karafka
       # Register all the listeners so they can be started and managed
       manager.register(listeners)
 
-      workers.each_with_index do |worker, i|
-        worker.async_call(
-          "karafka.worker##{i}",
-          worker_thread_priority
-        )
-      end
-
       # We aggregate threads here for a supervised shutdown process
-      Karafka::Server.workers = workers
       Karafka::Server.listeners = listeners
-      Karafka::Server.jobs_queue = jobs_queue
+
+      # Start worker threads after listeners are created so a failure in the boot steps above
+      # does not leave live worker threads blocked on an open queue.
+      workers.scale(Karafka::App.config.concurrency)
 
       until manager.done?
         conductor.wait
@@ -56,7 +51,7 @@ module Karafka
       # with everything. One thing worth keeping in mind though: It is the end user responsibility
       # to handle the shutdown detection in their long-running processes. Otherwise if timeout
       # is exceeded, there will be a forced shutdown.
-      workers.each(&:join)
+      workers.join
     # If anything crashes here, we need to raise the error and crush the runner because it means
     # that something terrible happened
     rescue => e
@@ -67,6 +62,11 @@ module Karafka
         type: "runner.call.error"
       )
       Karafka::App.stop!
+
+      # Clean up workers so we don't leak threads blocked on the queue
+      jobs_queue.close
+      workers.join
+
       raise e
     end
   end

data/lib/karafka/server.rb

@@ -10,7 +10,8 @@ module Karafka
       shutdown_timeout: %i[shutdown_timeout],
       forceful_exit_code: %i[internal forceful_exit_code],
       forceful_shutdown_wait: %i[internal forceful_shutdown_wait],
-      process: %i[internal process]
+      process: %i[internal process],
+      jobs_queue_class: %i[internal processing jobs_queue_class]
     )
 
     class << self
@@ -39,8 +40,7 @@ module Karafka
 
       # Method which runs app
       def run
-        self.listeners = []
-        self.workers = []
+        prepare
 
         # We need to validate this prior to running because it may be executed also from the
         # embedded
@@ -93,6 +93,7 @@ module Karafka
       # @note We don't need to sleep because Karafka::Runner is locking and waiting to finish loop
       # (and it won't happen until we explicitly want to stop)
       def start
+        prepare
         Karafka::Runner.new.call
       end
 
@@ -117,7 +118,7 @@ module Karafka
         # We divide it by 1000 because we use time in ms.
         ((timeout / 1_000) * (1 / supervision_sleep)).to_i.times do
           all_listeners_stopped = listeners.all?(&:stopped?)
-          all_workers_stopped = workers.none?(&:alive?)
+          all_workers_stopped = workers.stopped?
 
           return if all_listeners_stopped && all_workers_stopped
 
@@ -127,7 +128,7 @@ module Karafka
         raise Errors::ForcefulShutdownError
       rescue Errors::ForcefulShutdownError => e
         active_listeners = listeners.select(&:active?)
-        alive_workers = workers.select(&:alive?)
+        alive_workers = workers.alive
 
         # Collect details about subscription groups that still have jobs in processing
         in_processing = jobs_queue ? jobs_queue.in_processing : {}
@@ -143,7 +144,7 @@ module Karafka
         )
 
         # We're done waiting, lets kill them!
-        workers.each(&:terminate)
+        workers.terminate
         listeners.active.each(&:terminate)
 
         # We always need to shutdown clients to make sure we do not force the GC to close consumer.
@@ -186,9 +187,26 @@ module Karafka
           WaterDrop::ConnectionPool.close
 
           Karafka::App.terminate!
+
+          # Allow prepare to run again if the server is restarted (e.g. reset_status in tests,
+          # or embedded re-start).
+          @prepared = false
         end
       end
 
+      # Initializes listeners, jobs queue and workers pool.
+      # Called from both {.run} (standalone) and {.start} (embedded). Guarded so it runs only
+      # once even when {.run} delegates to {.start}.
+      def prepare
+        return if @prepared
+
+        @prepared = true
+
+        self.listeners = []
+        self.jobs_queue = jobs_queue_class.new
+        self.workers = Processing::WorkersPool.new
+      end
+
       # Quiets the Karafka server.
       #
       # Karafka will stop processing but won't quit the consumer group, so no rebalance will be
@@ -206,5 +224,9 @@ module Karafka
     self.execution_mode = ExecutionMode.new(:standalone)
 
     self.id = SecureRandom.hex(6)
+
+    # We need to have it set always for cases where it would be references outside of the Karafka
+    # process
+    self.workers = Processing::WorkersPool.new
   end
 end

data/lib/karafka/setup/attributes_map.rb

@@ -125,6 +125,7 @@ module Karafka
         sasl.oauthbearer.metadata.authentication.type
         sasl.oauthbearer.method
         sasl.oauthbearer.scope
+        sasl.oauthbearer.sub.claim.name
         sasl.oauthbearer.token.endpoint.url
         sasl.password
         sasl.username
@@ -283,6 +284,7 @@ module Karafka
         sasl.oauthbearer.metadata.authentication.type
         sasl.oauthbearer.method
         sasl.oauthbearer.scope
+        sasl.oauthbearer.sub.claim.name
         sasl.oauthbearer.token.endpoint.url
         sasl.password
         sasl.username

data/lib/karafka/setup/config.rb

@@ -225,6 +225,12 @@ module Karafka
           setting :activity_manager, default: Routing::ActivityManager.new
         end
 
+        # Namespace for declarative topics management (infrastructure-as-code)
+        setting :declaratives do
+          # option builder [Karafka::Declaratives::Builder] builder instance
+          setting :builder, default: Declaratives::Builder.new
+        end
+
         # Namespace for internal connection related settings
         setting :connection do
           # Manages starting up and stopping Kafka connections
@@ -305,24 +311,30 @@ module Karafka
           setting :jobs_queue_class, default: Processing::JobsQueue
           # option scheduler [Object] scheduler we will be using
           setting :scheduler_class, default: Processing::Schedulers::Default
-          # option jobs_builder [Object] jobs builder we want to use
-          setting :jobs_builder, default: Processing::JobsBuilder.new
-          # option coordinator [Class] work coordinator we want to user for processing coordination
-          setting :coordinator_class, default: Processing::Coordinator
-          # option errors_tracker_class [Class, nil] errors tracker that is used by the coordinator
-          #   for granular error tracking. `nil` for OSS as it is not in use.
-          setting :errors_tracker_class, default: nil
-          # option partitioner_class [Class] partitioner we use against a batch of data
-          setting :partitioner_class, default: Processing::Partitioner
-          # option strategy_selector [Object] processing strategy selector to be used
-          setting :strategy_selector, default: Processing::StrategySelector.new
-          # option expansions_selector [Object] processing expansions selector to be used
-          setting :expansions_selector, default: Processing::ExpansionsSelector.new
-          # option [Class] executor class
-          setting :executor_class, default: Processing::Executor
           # option worker_job_call_wrapper [Proc, false] callable object that will be used to wrap
           #   the worker execution of a job or false if no wrapper needed
           setting :worker_job_call_wrapper, default: false
+
+          # Consumer-group-specific processing defaults. When share groups land, a parallel
+          # `share_groups` namespace will hold their equivalents.
+          setting :consumer_groups do
+            # option jobs_builder [Object] jobs builder we want to use
+            setting :jobs_builder, default: Processing::ConsumerGroups::JobsBuilder.new
+            # option coordinator [Class] work coordinator we want to use for processing
+            #   coordination
+            setting :coordinator_class, default: Processing::ConsumerGroups::Coordinator
+            # option errors_tracker_class [Class, nil] errors tracker that is used by the
+            #   coordinator for granular error tracking. `nil` for OSS as it is not in use.
+            setting :errors_tracker_class, default: nil
+            # option partitioner_class [Class] partitioner we use against a batch of data
+            setting :partitioner_class, default: Processing::ConsumerGroups::Partitioner
+            # option strategy_selector [Object] processing strategy selector to be used
+            setting :strategy_selector, default: Processing::ConsumerGroups::StrategySelector.new
+            # option expansions_selector [Object] processing expansions selector to be used
+            setting :expansions_selector, default: Processing::ConsumerGroups::ExpansionsSelector.new
+            # option [Class] executor class
+            setting :executor_class, default: Processing::ConsumerGroups::Executor
+          end
         end
 
         # Things related to operating on messages
@@ -428,6 +440,11 @@ module Karafka
 
           configure_components(proxy)
 
+          # Install backwards-compatible forwarding so that gems (e.g. karafka-testing) that
+          # still access config.internal.processing.strategy_selector (etc.) keep working after
+          # the move to config.internal.processing.consumer_groups.*
+          install_processing_cg_forwarders(config)
+
           # Refreshes the references that are cached that might have been changed by the config
           Karafka.refresh!
 
@@ -446,6 +463,38 @@ module Karafka
 
         private
 
+        # Installs forwarding reader methods on the processing config node so that the old
+        # (pre-nesting) paths like `config.internal.processing.strategy_selector` still resolve
+        # by delegating to `config.internal.processing.consumer_groups.strategy_selector`.
+        # This keeps external gems (e.g. karafka-testing) working until they migrate.
+        #
+        # @param config [Karafka::Core::Configurable::Node] root config node
+        def install_processing_cg_forwarders(config)
+          processing = config.internal.processing
+          cg_node = processing.consumer_groups
+
+          %i[
+            jobs_builder
+            coordinator_class
+            errors_tracker_class
+            partitioner_class
+            strategy_selector
+            expansions_selector
+            executor_class
+          ].each do |setting_name|
+            writer = :"#{setting_name}="
+
+            # Remove previous definitions (if setup runs more than once) to avoid
+            # "method redefined" warnings that spec_helper promotes to errors
+            sc = processing.singleton_class
+            sc.remove_method(setting_name) if processing.respond_to?(setting_name)
+            sc.remove_method(writer) if processing.respond_to?(writer)
+
+            processing.define_singleton_method(setting_name) { cg_node.public_send(setting_name) }
+            processing.define_singleton_method(writer) { |val| cg_node.public_send(writer, val) }
+          end
+        end
+
         # Sets up all the components that are based on the user configuration
         # @param config_proxy [ConfigProxy] the configuration proxy containing deferred setup
         #   blocks

data/lib/karafka/setup/config_proxy.rb

@@ -45,8 +45,7 @@ module Karafka
     # Without the proxy, we'd have two problems:
     #
     # 1. **Permanent API pollution**: Adding a `producer` method to config that accepts blocks
-    #    would change its permanent API, even though this functionality is only needed during
-    #    setup.
+    #    would change its permanent API, even though this functionality is only needed during setup.
     #
     # 2. **Timing issues**: The producer doesn't exist yet when the user's setup block runs.
     #    The producer is created in `configure_components` after all user configuration is

data/lib/karafka/setup/contracts/config.rb

@@ -8,7 +8,7 @@ module Karafka
       #
       # @note There are many more configuration options inside of the
       #   `Karafka::Setup::Config` model, but we don't validate them here as they are
-      #   validated per each route (topic + consumer_group) because they can be overwritten,
+      #   validated per each route (topic + group) because they can be overwritten,
       #   so we validate all of that once all the routes are defined and ready.
       class Config < Karafka::Contracts::Base
         configure do |config|
@@ -121,16 +121,19 @@ module Karafka
           end
 
           nested(:processing) do
-            required(:jobs_builder) { |val| !val.nil? }
             required(:jobs_queue_class) { |val| !val.nil? }
             required(:scheduler_class) { |val| !val.nil? }
-            required(:coordinator_class) { |val| !val.nil? }
-            required(:errors_tracker_class) { |val| val.nil? || val.is_a?(Class) }
-            required(:partitioner_class) { |val| !val.nil? }
-            required(:strategy_selector) { |val| !val.nil? }
-            required(:expansions_selector) { |val| !val.nil? }
-            required(:executor_class) { |val| !val.nil? }
             required(:worker_job_call_wrapper) { |val| val == false || val.respond_to?(:wrap) }
+
+            nested(:consumer_groups) do
+              required(:jobs_builder) { |val| !val.nil? }
+              required(:coordinator_class) { |val| !val.nil? }
+              required(:errors_tracker_class) { |val| val.nil? || val.is_a?(Class) }
+              required(:partitioner_class) { |val| !val.nil? }
+              required(:strategy_selector) { |val| !val.nil? }
+              required(:expansions_selector) { |val| !val.nil? }
+              required(:executor_class) { |val| !val.nil? }
+            end
           end
 
           nested(:active_job) do
@@ -170,6 +173,23 @@ module Karafka
           detected_errors
         end
 
+        # Certain kafka settings are managed internally by Karafka and should not be set
+        # directly. Setting them manually may cause misbehaviours and other unexpected issues.
+        virtual do |data, errors|
+          next unless errors.empty?
+
+          managed_keys = Karafka::Setup::DefaultsInjector.managed_keys
+          detected_errors = []
+
+          data.fetch(:kafka).each_key do |key|
+            next unless managed_keys.include?(key)
+
+            detected_errors << [[:kafka, key], :managed_key_not_supported]
+          end
+
+          detected_errors
+        end
+
         virtual do |data, errors|
           next unless errors.empty?
 

data/lib/karafka/setup/defaults_injector.rb

@@ -52,6 +52,16 @@ module Karafka
       )
 
       class << self
+        # Kafka settings that are managed internally by Karafka and should not be set directly
+        # by users. Setting them manually may cause misbehaviours and other unexpected issues.
+        #
+        # @return [Set<Symbol>] set of managed kafka setting keys
+        def managed_keys
+          @managed_keys ||= Set[
+            :"statistics.unassigned.include"
+          ]
+        end
+
         # Propagates the kafka setting defaults unless they are already present for consumer config
         # This makes it easier to set some values that users usually don't change but still allows
         # them to overwrite the whole hash if they want to

data/lib/karafka/status.rb

@@ -41,8 +41,7 @@ module Karafka
       @status.to_s
     end
 
-    # Resets the status state
-    # This is used mostly in the integration suite
+    # Resets the status state This is used mostly in the integration suite
     def reset!
       @status = :initializing
     end

data/lib/karafka/swarm/liveness_listener.rb

@@ -19,6 +19,13 @@ module Karafka
         @mutex = Mutex.new
       end
 
+      # Report liveness before the first fetch loop iteration so the supervisor gets an initial
+      # healthy report even if the first consumption takes longer than the report timeout.
+      # @param _event [Karafka::Core::Monitoring::Event]
+      def on_connection_listener_before_fetch_loop(_event)
+        report_liveness
+      end
+
       # Report from the fetch loop at the top of each iteration
       # @param _event [Karafka::Core::Monitoring::Event]
       def on_connection_listener_fetch_loop(_event)

data/lib/karafka/swarm/manager.rb

@@ -19,12 +19,12 @@ module Karafka
         node_restart_timeout: %i[internal swarm node_restart_timeout]
       )
 
-      # Status we issue when we decide to shutdown unresponsive node
+      # Status we issue when we decide to shutdown a non-reporting node
       # We use -1 because nodes are expected to report 0+ statuses and we can use negative numbers
       # for non-node based statuses
-      NOT_RESPONDING_SHUTDOWN_STATUS = -1
+      NOT_REPORTING_SHUTDOWN_STATUS = -1
 
-      private_constant :NOT_RESPONDING_SHUTDOWN_STATUS
+      private_constant :NOT_REPORTING_SHUTDOWN_STATUS
 
       # @return [Array<Node>] All nodes that manager manages
       attr_reader :nodes
@@ -84,7 +84,7 @@ module Karafka
             if node.alive?
               next if terminate_if_hanging(statuses, node)
               next if stop_if_not_healthy(statuses, node)
-              next if stop_if_not_responding(statuses, node)
+              next if stop_if_not_reporting(statuses, node)
             else
               next if cleanup_one(statuses, node)
               next if restart_after_timeout(statuses, node)
@@ -144,12 +144,12 @@ module Karafka
         end
       end
 
-      # If node stopped responding, starts the stopping procedure.
+      # If node stopped reporting, starts the stopping procedure.
       #
       # @param statuses [Hash] hash with statuses transitions with times
       # @param node [Swarm::Node] node we're checking
       # @return [Boolean] should it be the last action taken on this node in this run
-      def stop_if_not_responding(statuses, node)
+      def stop_if_not_reporting(statuses, node)
         # Do nothing if already stopping
         return true if statuses.key?(:stop)
         # Do nothing if we've received status update recently enough
@@ -160,7 +160,7 @@ module Karafka
           "swarm.manager.stopping",
           caller: self,
           node: node,
-          status: NOT_RESPONDING_SHUTDOWN_STATUS
+          status: NOT_REPORTING_SHUTDOWN_STATUS
         ) do
           node.stop
           statuses[:stop] = monotonic_now

data/lib/karafka/swarm/node.rb

@@ -45,6 +45,14 @@ module Karafka
       #   - recreates producer and web producer
       # @note Parent API
       def start
+        # Close the previous reader pipe if it exists to prevent FD leaks on node restarts.
+        # Without this, each restart orphans the old reader pipe in the supervisor, and forked
+        # children inherit all accumulated leaked FDs.
+        begin
+          @reader&.close
+        rescue IOError
+          nil
+        end
         @reader, @writer = IO.pipe
         # Reset alive status when starting/restarting a node
         # nil means unknown status - will check with waitpid

data/lib/karafka/swarm/supervisor.rb

@@ -30,7 +30,15 @@ module Karafka
       # to reach out nodes and be processed to start the shutdown flow. Because of that and
       # because we always want to give all nodes all the time of `shutdown_timeout` they are
       # expected to have, we add this just to compensate.
-      SHUTDOWN_GRACE_PERIOD = 1_000
+      #
+      # Beyond signal propagation, child nodes also need a window after their own graceful
+      # `shutdown_timeout` to actually exit the process (run `at_exit` handlers, finalize
+      # librdkafka handles, close pools, etc.). On CI (especially macOS) the effective loop
+      # duration is noticeably longer than the nominal budget because `sleep` granularity and
+      # the per-iteration `waitpid` cost stretch each 0.1s tick. A too-tight grace period
+      # causes the supervisor to raise `ForcefulShutdownError` while a node is still in its
+      # final cleanup phase, which manifests as flaky shutdowns in swarm integration tests.
+      SHUTDOWN_GRACE_PERIOD = 15_000
 
       private_constant :SHUTDOWN_GRACE_PERIOD
 

data/lib/karafka/templates/karafka.rb.erb

@@ -94,14 +94,20 @@ class KarafkaApp < Karafka::App
     # active_job_topic :default
 <% end -%>
     topic :example do
-      # Uncomment this if you want Karafka to manage your topics configuration
-      # Managing topics configuration via routing will allow you to ensure config consistency
-      # across multiple environments
-      #
-      # config(partitions: 2, 'cleanup.policy': 'compact')
       consumer ExampleConsumer
     end
   end
+
+  # Uncomment this if you want Karafka to manage your topics configuration
+  # Declarative topics allow you to ensure topic config consistency across multiple environments
+  #
+  # declaratives.draw do
+  #   topic :example do
+  #     partitions 2
+  #     replication_factor 1
+  #     config 'cleanup.policy': 'compact'
+  #   end
+  # end
 end
 
 # Karafka now features a Web UI!

data/lib/karafka/version.rb

@@ -3,5 +3,5 @@
 # Main module namespace
 module Karafka
   # Current Karafka version
-  VERSION = "2.5.9"
+  VERSION = "2.6.0.beta1"
 end