karafka 2.5.9 → 2.6.0.beta1

data/lib/karafka/admin.rb

@@ -38,6 +38,21 @@ module Karafka
       @custom_kafka = kafka
     end
 
+    # No-op close to normalize the API surface.
+    #
+    # Each admin operation currently opens and closes its own underlying rdkafka admin instance
+    # internally, so there is nothing to release at the `Karafka::Admin` level right now. This
+    # method exists so that callers who hold an instance and call `#close` on it (matching the
+    # pattern of other closeable resources) do not raise `NoMethodError`.
+    #
+    # In the future, `Karafka::Admin` is planned to be refactored to reuse a single rdkafka admin
+    # instance across multiple operations rather than creating and tearing one down per call. When
+    # that happens, this method will need to release that shared instance. The no-op is here now
+    # so that all callers are already written against the correct API and require no changes when
+    # the real implementation lands.
+    def close
+    end
+
     class << self
       # Delegate topic-related operations to Topics class
 
@@ -80,18 +95,25 @@ module Karafka
         new.read_watermark_offsets(name_or_hash, partition)
       end
 
+      # @param topic_partition_offsets [Hash{String => Array<Hash>}] topics with partition specs
+      # @param isolation_level [Integer, nil] optional isolation level constant
+      # @see Topics.read_partition_offsets
+      def read_partition_offsets(topic_partition_offsets, isolation_level: nil)
+        new.read_partition_offsets(topic_partition_offsets, isolation_level: isolation_level)
+      end
+
       # @param topic_name [String] name of the topic we're interested in
       # @see Topics.info
       def topic_info(topic_name)
         new.topic_info(topic_name)
       end
 
-      # @param consumer_group_id [String] id of the consumer group for which we want to move the
+      # @param group_id [String] id of the group for which we want to move the
       #   existing offset
       # @param topics_with_partitions_and_offsets [Hash] Hash with list of topics and settings
       # @see ConsumerGroups.seek
-      def seek_consumer_group(consumer_group_id, topics_with_partitions_and_offsets)
-        new.seek_consumer_group(consumer_group_id, topics_with_partitions_and_offsets)
+      def seek_consumer_group(group_id, topics_with_partitions_and_offsets)
+        new.seek_consumer_group(group_id, topics_with_partitions_and_offsets)
       end
 
       # Takes consumer group and its topics and copies all the offsets to a new named group
@@ -124,36 +146,35 @@ module Karafka
         )
       end
 
-      # Removes given consumer group (if exists)
+      # Removes given group (if exists)
       #
-      # @param consumer_group_id [String] consumer group name
+      # @param group_id [String] group name
       # @see ConsumerGroups.delete
-      def delete_consumer_group(consumer_group_id)
-        new.delete_consumer_group(consumer_group_id)
+      def delete_consumer_group(group_id)
+        new.delete_consumer_group(group_id)
       end
 
-      # Triggers a rebalance for the specified consumer group
+      # Triggers a rebalance for the specified group
       #
-      # @param consumer_group_id [String] consumer group id to trigger rebalance for
+      # @param group_id [String] group id to trigger rebalance for
       # @see ConsumerGroups.trigger_rebalance
       # @note This API should be used only for development.
-      def trigger_rebalance(consumer_group_id)
-        new.trigger_rebalance(consumer_group_id)
+      def trigger_rebalance(group_id)
+        new.trigger_rebalance(group_id)
       end
 
-      # Reads lags and offsets for given topics in the context of consumer groups defined in the
-      #   routing
-      # @param consumer_groups_with_topics [Hash{String => Array<String>}] hash with consumer
-      #   groups names with array of topics to query per consumer group inside
+      # Reads lags and offsets for given topics in the context of groups defined in the routing
+      # @param groups_with_topics [Hash{String => Array<String>}] hash with group
+      #   names with array of topics to query per group inside
       # @param active_topics_only [Boolean] if set to false, when we use routing topics, will
       #   select also topics that are marked as inactive in routing
       # @return [Hash{String => Hash{Integer => Hash{Integer => Object}}}] hash where the top
-      #   level keys are the consumer groups and values are hashes with topics and inside
+      #   level keys are the groups and values are hashes with topics and inside
       #   partitions with lags and offsets
       # @see ConsumerGroups.read_lags_with_offsets
-      def read_lags_with_offsets(consumer_groups_with_topics = {}, active_topics_only: true)
+      def read_lags_with_offsets(groups_with_topics = {}, active_topics_only: true)
         new.read_lags_with_offsets(
-          consumer_groups_with_topics,
+          groups_with_topics,
           active_topics_only: active_topics_only
         )
       end
@@ -268,18 +289,25 @@ module Karafka
       Topics.new(kafka: @custom_kafka).read_watermark_offsets(name_or_hash, partition)
     end
 
+    # @param topic_partition_offsets [Hash{String => Array<Hash>}] topics with partition specs
+    # @param isolation_level [Integer, nil] optional isolation level constant
+    # @see Topics#read_partition_offsets
+    def read_partition_offsets(topic_partition_offsets, isolation_level: nil)
+      Topics.new(kafka: @custom_kafka).read_partition_offsets(topic_partition_offsets, isolation_level: isolation_level)
+    end
+
     # @param topic_name [String] name of the topic we're interested in
     # @see Topics#info
     def topic_info(topic_name)
       Topics.new(kafka: @custom_kafka).info(topic_name)
     end
 
-    # @param consumer_group_id [String] consumer group for which we want to move offsets
+    # @param group_id [String] group for which we want to move offsets
     # @param topics_with_partitions_and_offsets [Hash] hash with topics and settings
     # @see ConsumerGroups#seek
-    def seek_consumer_group(consumer_group_id, topics_with_partitions_and_offsets)
+    def seek_consumer_group(group_id, topics_with_partitions_and_offsets)
       ConsumerGroups.new(kafka: @custom_kafka).seek(
-        consumer_group_id,
+        group_id,
         topics_with_partitions_and_offsets
       )
     end
@@ -306,25 +334,25 @@ module Karafka
       )
     end
 
-    # @param consumer_group_id [String] consumer group name
+    # @param group_id [String] group name
     # @see ConsumerGroups#delete
-    def delete_consumer_group(consumer_group_id)
-      ConsumerGroups.new(kafka: @custom_kafka).delete(consumer_group_id)
+    def delete_consumer_group(group_id)
+      ConsumerGroups.new(kafka: @custom_kafka).delete(group_id)
     end
 
-    # @param consumer_group_id [String] consumer group id to trigger rebalance for
+    # @param group_id [String] group id to trigger rebalance for
     # @see ConsumerGroups#trigger_rebalance
-    def trigger_rebalance(consumer_group_id)
-      ConsumerGroups.new(kafka: @custom_kafka).trigger_rebalance(consumer_group_id)
+    def trigger_rebalance(group_id)
+      ConsumerGroups.new(kafka: @custom_kafka).trigger_rebalance(group_id)
     end
 
-    # @param consumer_groups_with_topics [Hash{String => Array<String>}] hash with consumer
-    #   groups names with array of topics
+    # @param groups_with_topics [Hash{String => Array<String>}] hash with group
+    #   names with array of topics
     # @param active_topics_only [Boolean] if set to false, will select also inactive topics
     # @see ConsumerGroups#read_lags_with_offsets
-    def read_lags_with_offsets(consumer_groups_with_topics = {}, active_topics_only: true)
+    def read_lags_with_offsets(groups_with_topics = {}, active_topics_only: true)
       ConsumerGroups.new(kafka: @custom_kafka).read_lags_with_offsets(
-        consumer_groups_with_topics,
+        groups_with_topics,
         active_topics_only: active_topics_only
       )
     end

data/lib/karafka/app.rb

@@ -37,18 +37,29 @@ module Karafka
         # We then narrow this to active consumer groups from which we select active subscription
         # groups.
         consumer_groups
-          .map { |cg| [cg, cg.subscription_groups] }
-          .select { |cg, _| cg.active? }
+          .map { |group| [group, group.subscription_groups] }
+          .select { |group, _| group.active? }
           .select { |_, sgs| sgs.delete_if { |sg| !sg.active? } }
           .delete_if { |_, sgs| sgs.empty? }
-          .each { |_, sgs| sgs.each { |sg| sg.topics.delete_if { |top| !top.active? } } }
+          .each { |_, sgs| sgs.each { |sg| sg.topics.delete_if { |topic| !topic.active? } } }
           .each { |_, sgs| sgs.delete_if { |sg| sg.topics.empty? } }
-          .reject { |cg, _| cg.subscription_groups.empty? }
+          .reject { |group, _| group.subscription_groups.empty? }
           .to_h
       end
 
+      # @return [Karafka::Declaratives::Builder] declaratives builder instance
+      def declaratives
+        config
+          .internal
+          .declaratives
+          .builder
+      end
+
       # Just a nicer name for the consumer groups
       alias_method :routes, :consumer_groups
+      # Generalized alias — routing entries are "groups" (consumer groups today, other kinds
+      # of groups like KIP-932 share groups in the future).
+      alias_method :groups, :consumer_groups
 
       # Returns current assignments of this process. Both topics and partitions
       #
@@ -108,7 +119,7 @@ module Karafka
         config.kafka[:debug] = contexts
         producer.config.kafka[:debug] = contexts
 
-        consumer_groups.map(&:topics).flat_map(&:to_a).each do |topic|
+        routes.map(&:topics).flat_map(&:to_a).each do |topic|
           topic.kafka[:debug] = contexts
         end
       end

data/lib/karafka/base_consumer.rb

@@ -26,7 +26,7 @@ module Karafka
     attr_accessor :messages
     # @return [Karafka::Connection::Client] kafka connection client
     attr_accessor :client
-    # @return [Karafka::Processing::Coordinator] coordinator
+    # @return [Karafka::Processing::ConsumerGroups::Coordinator] coordinator
     attr_accessor :coordinator
     # @return [Waterdrop::Producer] producer instance
     attr_accessor :producer
@@ -304,7 +304,7 @@ module Karafka
     # @return [Boolean] was this consumer in active use. Active use means running `#consume` at
     #   least once. Consumer may have to run `#revoked` or `#shutdown` despite not running
     #   `#consume` previously in delayed job cases and other cases that potentially involve running
-    #   the `Jobs::Idle` for house-keeping
+    #   the `ConsumerGroups::Jobs::Idle` for house-keeping
     def used?
       @used
     end

data/lib/karafka/cli/contracts/server.rb

@@ -16,7 +16,7 @@ module Karafka
           include
           exclude
         ].each do |action|
-          optional(:"#{action}_consumer_groups") { |cg| cg.is_a?(Array) }
+          optional(:"#{action}_consumer_groups") { |val| val.is_a?(Array) }
           optional(:"#{action}_subscription_groups") { |sg| sg.is_a?(Array) }
           optional(:"#{action}_topics") { |topics| topics.is_a?(Array) }
 
@@ -28,7 +28,7 @@ module Karafka
             # If there were no consumer_groups declared in the server cli, it means that we will
             # run all of them and no need to validate them here at all
             next if value.empty?
-            next if (value - Karafka::App.consumer_groups.map(&:name)).empty?
+            next if (value - Karafka::App.routes.map(&:name)).empty?
 
             # Found unknown consumer groups
             [[[:"#{action}_consumer_groups"], :consumer_groups_inclusion]]
@@ -44,7 +44,7 @@ module Karafka
             next if value.empty?
 
             subscription_groups = Karafka::App
-              .consumer_groups
+              .routes
               .map(&:subscription_groups)
               .flatten
               .map(&:name)
@@ -65,7 +65,7 @@ module Karafka
             next if value.empty?
 
             topics = Karafka::App
-              .consumer_groups
+              .routes
               .map(&:subscription_groups)
               .flatten
               .map(&:topics)

data/lib/karafka/cli/info.rb

@@ -41,7 +41,7 @@ module Karafka
           "Karafka version: #{Karafka::VERSION}#{postfix}",
           "Ruby version: #{RUBY_DESCRIPTION}",
           "Rdkafka version: #{::Rdkafka::VERSION}",
-          "Consumer groups count: #{Karafka::App.consumer_groups.size}",
+          "Consumer groups count: #{Karafka::App.routes.size}",
           "Subscription groups count: #{Karafka::App.subscription_groups.values.flatten.size}",
           "Workers count: #{concurrency}",
           "Instance client id: #{client_id}",

data/lib/karafka/cli/topics/base.rb

@@ -29,28 +29,20 @@ module Karafka
           raise Errors::CommandValidationError, cause: e
         end
 
-        # @return [Array<Karafka::Routing::Topic>] all available topics that can be managed
-        # @note If topic is defined in multiple consumer groups, first config will be used. This
-        #   means, that this CLI will not work for simultaneous management of multiple clusters
-        #   from a single CLI command execution flow.
+        # @return [Array<Karafka::Declaratives::Topic>] all available topics that can be managed
+        # @note Topics with bootstrap servers that differ from the default kafka config are
+        #   considered to belong to a different cluster and are excluded.
         def declaratives_routing_topics
-          return @declaratives_routing_topics if @declaratives_routing_topics
+          @declaratives_routing_topics ||= begin
+            default_servers = kafka_config[:"bootstrap.servers"]
 
-          collected_topics = {}
-          default_servers = kafka_config[:"bootstrap.servers"]
-
-          App.consumer_groups.each do |consumer_group|
-            consumer_group.topics.each do |topic|
-              # Skip topics that were explicitly disabled from management
-              next unless topic.declaratives.active?
-              # If bootstrap servers are different, consider this a different cluster
-              next unless default_servers == topic.kafka[:"bootstrap.servers"]
-
-              collected_topics[topic.name] ||= topic
+            App.declaratives.topics.reject do |topic|
+              # Topics with explicit bootstrap servers that differ from the default are on a
+              # different cluster. Topics without bootstrap_servers set (e.g. from the standalone
+              # DSL) are assumed to belong to the default cluster.
+              topic.bootstrap_servers && topic.bootstrap_servers != default_servers
             end
           end
-
-          @declaratives_routing_topics = collected_topics.values
         end
 
         # @return [Array<Hash>] existing topics details

data/lib/karafka/cli/topics/repartition.rb

@@ -17,7 +17,7 @@ module Karafka
           declaratives_routing_topics.each do |topic|
             name = topic.name
 
-            desired_count = topic.config.partitions
+            desired_count = topic.declaratives.partitions
             existing_count = existing_partitions.fetch(name, false)
 
             if existing_count && existing_count < desired_count

data/lib/karafka/connection/client.rb

@@ -70,8 +70,11 @@ module Karafka
         @closed = false
         @subscription_group = subscription_group
         @buffer = RawMessagesBuffer.new
-        @rebalance_manager = RebalanceManager.new(@subscription_group.id, @buffer)
-        @rebalance_callback = Instrumentation::Callbacks::Rebalance.new(@subscription_group, id)
+        @rebalance_manager = ConsumerGroups::RebalanceManager.new(@subscription_group.id, @buffer)
+        @rebalance_callback = Instrumentation::Callbacks::ConsumerGroups::Rebalance.new(
+          @subscription_group,
+          id
+        )
         @mode = Mode.new
 
         @interval_runner = Helpers::IntervalRunner.new do
@@ -401,10 +404,14 @@ module Karafka
       #   - OAUTHBEARER token refresh callbacks
       #
       # @param timeout [Integer] number of milliseconds to wait on events or 0 not to wait.
+      # @param safe [Boolean] when true, rescues Rdkafka::RdkafkaError so callers in
+      #   shutdown/quiet paths do not trigger a full listener reset. When shutting down, errors
+      #   at this layer are not relevant enough. We want to log them but we should not propagate
+      #   them any further.
       #
       # @note It is non-blocking when timeout 0 and will not wait if queue empty. It costs up to
       #   2ms when no callbacks are triggered.
-      def events_poll(timeout = 0)
+      def events_poll(timeout = 0, safe: false)
         kafka.events_poll(timeout)
 
         # Emit event for monitoring - happens once per tick_interval (default 5s)
@@ -414,6 +421,8 @@ module Karafka
           caller: self,
           subscription_group: @subscription_group
         )
+      rescue Rdkafka::RdkafkaError
+        safe ? nil : raise
       end
 
       # Returns pointer to the consumer group metadata. It is used only in the context of
@@ -727,9 +736,9 @@ module Karafka
         # Register statistics runner for this particular type of callbacks
         Karafka::Core::Instrumentation.statistics_callbacks.add(
           @subscription_group.id,
-          Instrumentation::Callbacks::Statistics.new(
+          Instrumentation::Callbacks::ConsumerGroups::Statistics.new(
             @subscription_group.id,
-            @subscription_group.consumer_group.id,
+            @subscription_group.group.id,
             @name
           )
         )
@@ -737,9 +746,9 @@ module Karafka
         # Register error tracking callback
         Karafka::Core::Instrumentation.error_callbacks.add(
           @subscription_group.id,
-          Instrumentation::Callbacks::Error.new(
+          Instrumentation::Callbacks::ConsumerGroups::Error.new(
             @subscription_group.id,
-            @subscription_group.consumer_group.id,
+            @subscription_group.group.id,
             @name
           )
         )
@@ -765,6 +774,29 @@ module Karafka
         end
 
         consumer
+      rescue
+        # If anything past the consumer allocation raises (e.g., subscribe/assign failing due to
+        # DNS, unknown_topic_or_part, unreleased_instance_id for static group membership,
+        # transient broker issues, etc.), we must clean up the native rdkafka handle and the
+        # callback registry entries we just added. Otherwise the oauth callback keeps a strong
+        # reference to the consumer, pinning it from GC, and the native rd_kafka_t plus its
+        # internal pipe FDs and housekeeping threads leak. Callers up the stack (the listener
+        # rescue + reset loop) will simply retry build_consumer.
+        sg_id = @subscription_group.id
+        Karafka::Core::Instrumentation.statistics_callbacks.delete(sg_id)
+        Karafka::Core::Instrumentation.error_callbacks.delete(sg_id)
+        Karafka::Core::Instrumentation.oauthbearer_token_refresh_callbacks.delete(sg_id)
+
+        # Close safely if the consumer was allocated. Native kafka was never started
+        # (native_kafka_auto_start: false and we did not reach @kafka.start), so close only
+        # destroys the inner handle without needing to join any polling thread.
+        begin
+          consumer&.close
+        rescue
+          nil
+        end
+
+        raise
       end
 
       # @return [Rdkafka::Consumer] librdkafka consumer instance
@@ -783,8 +815,7 @@ module Karafka
       # Decides whether or not we should unsubscribe prior to closing.
       #
       # We cannot do it when there is a static group membership assignment as it would be
-      # reassigned.
-      # We cannot do it also for assign mode because then there are no subscriptions
+      # reassigned. We cannot do it also for assign mode because then there are no subscriptions
       # We also do not do it if there are no assignments at all as it does not make sense
       #
       # @return [Boolean] should we unsubscribe prior to shutdown

data/lib/karafka/connection/consumer_groups/rebalance_manager.rb

@@ -0,0 +1,120 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Connection
+    # Consumer-group-specific connection components (rebalance handling driven by consumer group
+    # protocol). Parallel `ShareGroups` will live next to this namespace once KIP-932 lands.
+    module ConsumerGroups
+      # Manager for tracking changes in the partitions assignment after the assignment is done and
+      # for ensuring, that proper buffer related operations that may be impacted by the rebalance
+      # state are applied.
+      #
+      # We need tracking of those to clean up consumers that will no longer process given
+      # partitions as they were taken away.
+      #
+      # @note Since this does not happen really often, we try to stick with same objects for the
+      #   empty states most of the time, so we don't create many objects during the manager life
+      #
+      # @note Internally in the rebalance manager we have a notion of lost partitions. Partitions
+      #   that are lost, are those that got revoked but did not get re-assigned back. We do not
+      #   expose this concept outside and we normalize to have them revoked, as it is irrelevant
+      #   from the rest of the code perspective as only those that are lost are truly revoked.
+      #
+      # @note For cooperative-sticky `#assigned_partitions` holds only the recently assigned
+      #   partitions, not all the partitions that are owned
+      #
+      # @note We have to have the `subscription_group` reference because we have a global pipeline
+      #   for notifications and we need to make sure we track changes only for things that are of
+      #   relevance to our subscription group
+      class RebalanceManager
+        # Empty array for internal usage not to create new objects
+        EMPTY_ARRAY = [].freeze
+
+        attr_reader :assigned_partitions, :revoked_partitions
+
+        private_constant :EMPTY_ARRAY
+
+        # @param subscription_group_id [String] subscription group id
+        # @param buffer [Karafka::Connection::RawMessagesBuffer]
+        # @return [RebalanceManager]
+        def initialize(subscription_group_id, buffer)
+          @assigned_partitions = {}
+          @revoked_partitions = {}
+          @changed = false
+          @active = false
+          @subscription_group_id = subscription_group_id
+          @buffer = buffer
+
+          # Connects itself to the instrumentation pipeline so rebalances can be tracked
+          Karafka.monitor.subscribe(self)
+        end
+
+        # Resets the rebalance manager state
+        # This needs to be done before each polling loop as during the polling, the state may be
+        # changed
+        def clear
+          @assigned_partitions.clear
+          @revoked_partitions.clear
+          @changed = false
+        end
+
+        # @return [Boolean] indicates a state change in the partitions assignment
+        def changed?
+          @changed
+        end
+
+        # @return [Boolean] true if there was at least one rebalance
+        # @note This method is needed to make sure that when using cooperative-sticky, we do not
+        #   close until first rebalance. Otherwise librdkafka may crash.
+        # @see https://github.com/confluentinc/librdkafka/issues/4312
+        def active?
+          @active
+        end
+
+        # Callback that kicks in inside of rdkafka, when new partitions were assigned.
+        #
+        # @private
+        # @param event [Karafka::Core::Monitoring::Event]
+        def on_rebalance_partitions_assigned(event)
+          # Apply changes only for our subscription group
+          return unless event[:subscription_group_id] == @subscription_group_id
+
+          @active = true
+          @assigned_partitions = event[:tpl].to_h.transform_values { |part| part.map(&:partition) }
+          @changed = true
+        end
+
+        # Callback that kicks in inside of rdkafka, when partitions were revoked.
+        #
+        # @private
+        # @param event [Karafka::Core::Monitoring::Event]
+        def on_rebalance_partitions_revoked(event)
+          # Apply changes only for our subscription group
+          return unless event[:subscription_group_id] == @subscription_group_id
+
+          @active = true
+          @revoked_partitions = event[:tpl].to_h.transform_values { |part| part.map(&:partition) }
+          @changed = true
+
+          remove_revoked_and_duplicated_messages
+        end
+
+        private
+
+        # We may have a case where in the middle of data polling, we've lost a partition.
+        # In a case like this we should remove all the pre-buffered messages from list partitions
+        # as we are no longer responsible in a given process for processing those messages and
+        # they should have been picked up by a different process.
+        def remove_revoked_and_duplicated_messages
+          @revoked_partitions.each do |topic, partitions|
+            partitions.each do |partition|
+              @buffer.delete(topic, partition)
+            end
+          end
+
+          @buffer.uniq!
+        end
+      end
+    end
+  end
+end

data/lib/karafka/connection/listener.rb

@@ -13,8 +13,8 @@ module Karafka
       include Helpers::Async
 
       include Helpers::ConfigImporter.new(
-        jobs_builder: %i[internal processing jobs_builder],
-        partitioner_class: %i[internal processing partitioner_class],
+        jobs_builder: %i[internal processing consumer_groups jobs_builder],
+        partitioner_class: %i[internal processing consumer_groups partitioner_class],
         reset_backoff: %i[internal connection reset_backoff],
         listener_thread_priority: %i[internal connection listener_thread_priority]
       )
@@ -26,7 +26,7 @@ module Karafka
       # @return [Karafka::Routing::SubscriptionGroup] subscription group that this listener handles
       attr_reader :subscription_group
 
-      # @return [Processing::CoordinatorsBuffer] coordinator buffers that can be used directly in
+      # @return [Processing::ConsumerGroups::CoordinatorsBuffer] coordinator buffers that can be used directly in
       #   advanced cases of changes to the polling flow (like triggered seek back without messages
       #   ahead in the topic)
       attr_reader :coordinators
@@ -45,12 +45,12 @@ module Karafka
         @id = SecureRandom.hex(6)
         @subscription_group = subscription_group
         @jobs_queue = jobs_queue
-        @coordinators = Processing::CoordinatorsBuffer.new(subscription_group.topics)
+        @coordinators = Processing::ConsumerGroups::CoordinatorsBuffer.new(subscription_group.topics)
         @client = Client.new(@subscription_group, -> { running? })
-        @executors = Processing::ExecutorsBuffer.new(@client, subscription_group)
+        @executors = Processing::ConsumerGroups::ExecutorsBuffer.new(@client, subscription_group)
         @partitioner = partitioner_class.new(subscription_group)
         @scheduler = scheduler
-        @events_poller = Helpers::IntervalRunner.new { @client.events_poll }
+        @events_poller = Helpers::IntervalRunner.new { |**opts| @client.events_poll(**opts) }
         # We keep one buffer for messages to preserve memory and not allocate extra objects
         # We can do this that way because we always first schedule jobs using messages before we
         # fetch another batch.
@@ -499,6 +499,7 @@ module Karafka
         until wait_until.call
           @client.ping
           @scheduler.on_manage
+          @events_poller.call(safe: true)
 
           after_ping.call
           sleep(0.2)
@@ -520,7 +521,7 @@ module Karafka
         @client.reset
         @coordinators.reset
         @interval_runner.reset
-        @executors = Processing::ExecutorsBuffer.new(@client, @subscription_group)
+        @executors = Processing::ConsumerGroups::ExecutorsBuffer.new(@client, @subscription_group)
       end
     end
   end

data/lib/karafka/connection/listeners_batch.rb

@@ -13,7 +13,7 @@ module Karafka
         # should be able to distribute work whenever any work is done in any of the listeners
         scheduler = App.config.internal.processing.scheduler_class.new(jobs_queue)
 
-        @batch = App.subscription_groups.flat_map do |_consumer_group, subscription_groups|
+        @batch = App.subscription_groups.flat_map do |_group, subscription_groups|
           subscription_groups.map do |subscription_group|
             Connection::Listener.new(
               subscription_group,

data/lib/karafka/connection/mode.rb

@@ -31,8 +31,7 @@ module Karafka
       end
 
       # Define query and setter methods for each mode using meta-programming
-      # This creates methods like: subscribe?, assign?
-      # And bang methods like: subscribe!, assign!
+      # This creates methods like: subscribe?, assign? And bang methods like: subscribe!, assign!
       MODES.each do |mode_name|
         # @return [Boolean] true if the current mode matches this mode
         define_method("#{mode_name}?") do

data/lib/karafka/connection/raw_messages_buffer.rb

@@ -108,11 +108,6 @@ module Karafka
       #   we save ourselves some objects allocations. We cannot clear the underlying arrays as they
       #   may be used in other threads for data processing, thus if we would clear it, we could
       #   potentially clear a raw messages array for a job that is in the jobs queue.
-      #
-      # @note We do not clear the eof assignments because they can span across batch pollings.
-      #   Since eof is not raised non-stop and is silenced after an eof poll, if we would clean it
-      #   here we would loose the notion of it. The reset state for it should happen when we do
-      #   discover new messages for given topic partition.
       def clear
         @size = 0
         @groups.each_value(&:clear)