karafka 2.5.1 → 2.5.2

data/lib/karafka/admin/consumer_groups.rb

@@ -234,6 +234,76 @@ module Karafka
           end
         end
 
+        # Triggers a rebalance for the specified consumer group by briefly joining and leaving
+        #
+        # @param consumer_group_id [String] consumer group id to trigger rebalance for
+        #
+        # @return [void]
+        #
+        # @raise [Karafka::Errors::InvalidConfigurationError] when consumer group is not found in
+        #   routing or has no topics
+        #
+        # @note This method creates a temporary "fake" consumer that joins the consumer group,
+        #   triggering a rebalance when it joins and another when it leaves. This should only be
+        #   used for operational/testing purposes as it causes two rebalances.
+        #
+        # @note The consumer group does not need to be running for this to work, but if it is,
+        #   it will experience rebalances.
+        #
+        # @note The behavior follows the configured rebalance protocol. For cooperative sticky
+        #   rebalancing or KIP-848 based protocols, there may be no immediate reaction to the
+        #   rebalance trigger as these protocols allow incremental partition reassignments without
+        #   stopping all consumers.
+        #
+        # @note Topics are always detected from the routing configuration. The consumer settings
+        #   (kafka config) are taken from the first topic in the consumer group to ensure
+        #   consistency with the actual consumer configuration.
+        #
+        # @example Trigger rebalance for a consumer group
+        #   Karafka::Admin::ConsumerGroups.trigger_rebalance('my-group')
+        def trigger_rebalance(consumer_group_id)
+          consumer_group = Karafka::App.routes.find { |cg| cg.id == consumer_group_id }
+
+          unless consumer_group
+            raise(
+              Errors::InvalidConfigurationError,
+              "Consumer group '#{consumer_group_id}' not found in routing"
+            )
+          end
+
+          topics = consumer_group.topics.map(&:name)
+
+          if topics.empty?
+            raise(
+              Errors::InvalidConfigurationError,
+              "Consumer group '#{consumer_group_id}' has no topics"
+            )
+          end
+
+          # Get the first topic to extract kafka settings
+          first_topic = consumer_group.topics.first
+
+          # Build consumer settings using the consumer group's kafka config from first topic
+          # This ensures we use the same settings as the actual consumers
+          # Following the same pattern as in Karafka::Connection::Client#build_kafka
+          consumer_settings = Setup::AttributesMap.consumer(first_topic.kafka.dup)
+          consumer_settings[:'group.id'] = consumer_group.id
+          consumer_settings[:'enable.auto.offset.store'] = false
+          consumer_settings[:'auto.offset.reset'] ||= first_topic.initial_offset
+
+          with_consumer(consumer_settings) do |consumer|
+            # Subscribe to the topics - this triggers the first rebalance
+            consumer.subscribe(*topics)
+
+            # Wait briefly (100ms) to allow the rebalance to initiate
+            # The actual rebalance happens asynchronously, so we just need to give it a moment
+            sleep(0.1)
+
+            # Unsubscribe - this will trigger the second rebalance when the consumer closes
+            # The ensure block in with_consumer will handle the unsubscribe and close
+          end
+        end
+
         # Reads lags and offsets for given topics in the context of consumer groups defined in the
         #   routing
         #
@@ -257,19 +327,19 @@ module Karafka
           # We first fetch all the topics with partitions count that exist in the cluster so we
           # do not query for topics that do not exist and so we can get partitions count for all
           # the topics we may need. The non-existent and not consumed will be filled at the end
-          existing_topics = cluster_info.topics.map do |topic|
+          existing_topics = cluster_info.topics.to_h do |topic|
             [topic[:topic_name], topic[:partition_count]]
-          end.to_h.freeze
+          end.freeze
 
           # If no expected CGs, we use all from routing that have active topics
           if consumer_groups_with_topics.empty?
-            consumer_groups_with_topics = Karafka::App.routes.map do |cg|
+            consumer_groups_with_topics = Karafka::App.routes.to_h do |cg|
               cg_topics = cg.topics.select do |cg_topic|
                 active_topics_only ? cg_topic.active? : true
               end
 
               [cg.id, cg_topics.map(&:name)]
-            end.to_h
+            end
           end
 
           # We make a copy because we will remove once with non-existing topics

data/lib/karafka/admin/topics.rb

@@ -36,7 +36,7 @@ module Karafka
 
             # Build the requested range - since first element is on the start offset we need to
             # subtract one from requested count to end up with expected number of elements
-            requested_range = (start_offset..start_offset + (count - 1))
+            requested_range = (start_offset..(start_offset + (count - 1)))
             # Establish theoretical available range. Note, that this does not handle cases related
             # to log retention or compaction
             available_range = (low_offset..(high_offset - 1))
@@ -139,15 +139,48 @@ module Karafka
           end
         end
 
-        # Fetches the watermark offsets for a given topic partition
+        # Fetches the watermark offsets for a given topic partition or multiple topics and
+        # partitions
         #
-        # @param name [String, Symbol] topic name
-        # @param partition [Integer] partition
-        # @return [Array<Integer, Integer>] low watermark offset and high watermark offset
-        def read_watermark_offsets(name, partition)
+        # @param name_or_hash [String, Symbol, Hash] topic name or hash with topics and partitions
+        # @param partition [Integer, nil] partition number (required when first param is topic name)
+        #
+        # @return [Array<Integer, Integer>, Hash] when querying single partition returns array with
+        #   low and high watermark offsets, when querying multiple returns nested hash
+        #
+        # @example Query single partition
+        #   Karafka::Admin::Topics.read_watermark_offsets('events', 0)
+        #   # => [0, 100]
+        #
+        # @example Query specific partitions across multiple topics
+        #   Karafka::Admin::Topics.read_watermark_offsets(
+        #     { 'events' => [0, 1], 'logs' => [0] }
+        #   )
+        #   # => {
+        #   #   'events' => {
+        #   #     0 => [0, 100],
+        #   #     1 => [0, 150]
+        #   #   },
+        #   #   'logs' => {
+        #   #     0 => [0, 50]
+        #   #   }
+        #   # }
+        def read_watermark_offsets(name_or_hash, partition = nil)
+          # Normalize input to hash format
+          topics_with_partitions = partition ? { name_or_hash => [partition] } : name_or_hash
+
+          result = Hash.new { |h, k| h[k] = {} }
+
           with_consumer do |consumer|
-            consumer.query_watermark_offsets(name, partition)
+            topics_with_partitions.each do |topic, partitions|
+              partitions.each do |partition_id|
+                result[topic][partition_id] = consumer.query_watermark_offsets(topic, partition_id)
+              end
+            end
           end
+
+          # Return single array for single partition query, hash for multiple
+          partition ? result.dig(name_or_hash, partition) : result
         end
 
         # Returns basic topic metadata

data/lib/karafka/admin.rb

@@ -59,11 +59,11 @@ module Karafka
         Topics.create_partitions(name, partitions)
       end
 
-      # @param name [String, Symbol] topic name
-      # @param partition [Integer] partition
+      # @param name_or_hash [String, Symbol, Hash] topic name or hash with topics and partitions
+      # @param partition [Integer, nil] partition (nil when using hash format)
       # @see Topics.read_watermark_offsets
-      def read_watermark_offsets(name, partition)
-        Topics.read_watermark_offsets(name, partition)
+      def read_watermark_offsets(name_or_hash, partition = nil)
+        Topics.read_watermark_offsets(name_or_hash, partition)
       end
 
       # @param topic_name [String] name of the topic we're interested in
@@ -113,6 +113,15 @@ module Karafka
         ConsumerGroups.delete(consumer_group_id)
       end
 
+      # Triggers a rebalance for the specified consumer group
+      #
+      # @param consumer_group_id [String] consumer group id to trigger rebalance for
+      # @see ConsumerGroups.trigger_rebalance
+      # @note This API should be used only for development.
+      def trigger_rebalance(consumer_group_id)
+        ConsumerGroups.trigger_rebalance(consumer_group_id)
+      end
+
       # Reads lags and offsets for given topics in the context of consumer groups defined in the
       #   routing
       # @param consumer_groups_with_topics [Hash<String, Array<String>>] hash with consumer groups

data/lib/karafka/base_consumer.rb

@@ -14,8 +14,9 @@ module Karafka
 
     def_delegators :@coordinator, :topic, :partition, :eofed?, :seek_offset, :seek_offset=
 
-    def_delegators :producer, :produce_async, :produce_sync, :produce_many_async,
-                   :produce_many_sync
+    def_delegators(
+      :producer, :produce_async, :produce_sync, :produce_many_async, :produce_many_sync
+    )
 
     def_delegators :messages, :each
 
@@ -81,9 +82,8 @@ module Karafka
     # @private
     #
     # @param action [Symbol]
-    # @param block [Proc]
-    def on_wrap(action, &block)
-      handle_wrap(action, &block)
+    def on_wrap(action, &)
+      handle_wrap(action, &)
     rescue StandardError => e
       monitor.instrument(
         'error.occurred',

data/lib/karafka/cli/base.rb

@@ -68,7 +68,7 @@ module Karafka
 
           # However when it is unavailable, we still want to be able to run help command
           # and install command as they don't require configured app itself to run
-          return if %w[-h install].any? { |cmd| cmd == ARGV[0] }
+          return if %w[-h install].any?(ARGV[0])
 
           # All other commands except help and install do require an existing boot file if it was
           # declared

data/lib/karafka/cli/contracts/server.rb

@@ -7,10 +7,8 @@ module Karafka
       # Contract for validating correctness of the server cli command options.
       class Server < ::Karafka::Contracts::Base
         configure do |config|
-          config.error_messages = YAML.safe_load(
-            File.read(
-              File.join(Karafka.gem_root, 'config', 'locales', 'errors.yml')
-            )
+          config.error_messages = YAML.safe_load_file(
+            File.join(Karafka.gem_root, 'config', 'locales', 'errors.yml')
           ).fetch('en').fetch('validations').fetch('cli').fetch('server')
         end
 

data/lib/karafka/cli/install.rb

@@ -54,7 +54,7 @@ module Karafka
           template = File.read(Karafka.core_root.join("templates/#{source}"))
           render = ::ERB.new(template, trim_mode: '-').result(binding)
 
-          File.open(pathed_target, 'w') { |file| file.write(render) }
+          File.write(pathed_target, render)
 
           puts "#{green('Created')} #{target}"
         end

data/lib/karafka/cli/topics/align.rb

@@ -95,7 +95,7 @@ module Karafka
               names = config.synonyms.map(&:name) << config.name
 
               # We move forward only if given topic config is for altering
-              next if (desired_configs.keys & names).empty?
+              next unless desired_configs.keys.intersect?(names)
 
               desired_config = nil
 

data/lib/karafka/cli/topics/repartition.rb

@@ -10,9 +10,9 @@ module Karafka
         def call
           any_repartitioned = false
 
-          existing_partitions = existing_topics.map do |topic|
+          existing_partitions = existing_topics.to_h do |topic|
             [topic.fetch(:topic_name), topic.fetch(:partition_count)]
-          end.to_h
+          end
 
           declaratives_routing_topics.each do |topic|
             name = topic.name

data/lib/karafka/connection/client.rb

@@ -51,7 +51,9 @@ module Karafka
         :fenced, # -144
         :auto_offset_reset, # -140
         # This can happen for many reasons, including issues with static membership being fenced
-        :fatal # -150
+        :fatal, # -150,
+        # This can happen with new rebalance protocol and same group.instance.id in use
+        :unreleased_instance_id # 111
       ].freeze
 
       private_constant :MAX_POLL_RETRIES, :COOP_UNSUBSCRIBE_FACTOR, :EARLY_REPORT_ERRORS
@@ -407,6 +409,14 @@ module Karafka
       #   2ms when no callbacks are triggered.
       def events_poll(timeout = 0)
         kafka.events_poll(timeout)
+
+        # Emit event for monitoring - happens once per tick_interval (default 5s)
+        # Listeners can check assignment_lost?, track polling health, etc.
+        Karafka.monitor.instrument(
+          'client.events_poll',
+          caller: self,
+          subscription_group: @subscription_group
+        )
       end
 
       # Returns pointer to the consumer group metadata. It is used only in the context of
@@ -612,7 +622,7 @@ module Karafka
         # We should not run a single poll longer than the tick frequency. Otherwise during a single
         # `#batch_poll` we would not be able to run `#events_poll` often enough effectively
         # blocking events from being handled.
-        poll_tick = timeout > tick_interval ? tick_interval : timeout
+        poll_tick = [timeout, tick_interval].min
 
         result = kafka.poll(poll_tick)
 

data/lib/karafka/connection/listeners_batch.rb

@@ -25,9 +25,8 @@ module Karafka
       end
 
       # Iterates over available listeners and yields each listener
-      # @param block [Proc] block we want to run
-      def each(&block)
-        @batch.each(&block)
+      def each(&)
+        @batch.each(&)
       end
 
       # @return [Array<Listener>] active listeners

data/lib/karafka/connection/proxy.rb

@@ -10,6 +10,10 @@ module Karafka
     # do still want to be able to alter some functionalities. This wrapper helps us do it when
     # it would be needed
     class Proxy < SimpleDelegator
+      include Helpers::ConfigImporter.new(
+        proxy_config: %i[internal connection proxy]
+      )
+
       # Errors on which we want to retry
       # Includes temporary errors related to node not being (or not yet being) coordinator or a
       # leader to a given set of partitions. Usually goes away after a retry
@@ -21,6 +25,7 @@ module Karafka
         not_coordinator
         not_leader_for_partition
         coordinator_load_in_progress
+        stale_member_epoch
       ].freeze
 
       private_constant :RETRYABLE_DEFAULT_ERRORS
@@ -36,7 +41,6 @@ module Karafka
         # wrap an already wrapped object with another proxy level. Simplifies passing consumers
         # and makes it safe to wrap without type checking
         @wrapped = obj.is_a?(self.class) ? obj.wrapped : obj
-        @config = ::Karafka::App.config.internal.connection.proxy
       end
 
       # Proxies the `#query_watermark_offsets` with extra recovery from timeout problems.
@@ -47,7 +51,7 @@ module Karafka
       # @param partition [Integer] partition number
       # @return [Array<Integer, Integer>] watermark offsets
       def query_watermark_offsets(topic, partition)
-        l_config = @config.query_watermark_offsets
+        l_config = proxy_config.query_watermark_offsets
 
         # For newly created topics or in cases where we're trying to get them but there is no
         # leader, this can fail. It happens more often for new topics under KRaft, however we
@@ -67,7 +71,7 @@ module Karafka
       # @param tpl [Rdkafka::Consumer::TopicPartitionList] tpl to get time offsets
       # @return [Rdkafka::Consumer::TopicPartitionList] tpl with time offsets
       def offsets_for_times(tpl)
-        l_config = @config.offsets_for_times
+        l_config = proxy_config.offsets_for_times
 
         with_broker_errors_retry(
           # required to be in seconds, not ms
@@ -84,7 +88,7 @@ module Karafka
       #   assignment tpl usage
       # @return [Rdkafka::Consumer::TopicPartitionList] tpl with committed offsets and metadata
       def committed(tpl = nil)
-        c_config = @config.committed
+        c_config = proxy_config.committed
 
         with_broker_errors_retry(
           # required to be in seconds, not ms
@@ -121,7 +125,7 @@ module Karafka
       #   even when no stored, because with sync commit, it refreshes the ownership state of the
       #   consumer in a sync way.
       def commit_offsets(tpl = nil, async: true)
-        c_config = @config.commit
+        c_config = proxy_config.commit
 
         with_broker_errors_retry(
           wait_time: c_config.wait_time / 1_000.to_f,
@@ -153,7 +157,7 @@ module Karafka
       #   we want to get the lag on the defined CG
       # @return [Hash<String, Hash>] hash with topics and their partitions lags
       def lag(tpl)
-        l_config = @config.committed
+        l_config = proxy_config.committed
 
         with_broker_errors_retry(
           # required to be in seconds, not ms
@@ -168,7 +172,7 @@ module Karafka
       #   get info on all topics
       # @return [Rdkafka::Metadata] rdkafka metadata object with the requested details
       def metadata(topic_name = nil)
-        m_config = @config.metadata
+        m_config = proxy_config.metadata
 
         with_broker_errors_retry(
           # required to be in seconds, not ms

data/lib/karafka/env.rb

@@ -21,8 +21,7 @@ module Karafka
       super('')
 
       LOOKUP_ENV_KEYS
-        .map { |key| ENV[key] }
-        .compact
+        .filter_map { |key| ENV.fetch(key, nil) }
         .first
         .then { |env| env || DEFAULT_ENV }
         .then { |env| replace(env) }

data/lib/karafka/helpers/interval_runner.rb

@@ -11,11 +11,14 @@ module Karafka
     # or other places but would only slow things down if would run with each tick.
     class IntervalRunner
       include Karafka::Core::Helpers::Time
+      include Helpers::ConfigImporter.new(
+        tick_interval: %i[internal tick_interval]
+      )
 
       # @param interval [Integer] interval in ms for running the provided code. Defaults to the
       #   `internal.tick_interval` value
       # @param block [Proc] block of code we want to run once in a while
-      def initialize(interval: ::Karafka::App.config.internal.tick_interval, &block)
+      def initialize(interval: tick_interval, &block)
         @block = block
         @interval = interval
         @last_called_at = monotonic_now - @interval

data/lib/karafka/instrumentation/assignments_tracker.rb

@@ -79,6 +79,23 @@ module Karafka
         end
       end
 
+      # Handles events_poll notification to detect assignment loss
+      # This is called regularly (every tick_interval) so we check if assignment was lost
+      #
+      # @param event [Karafka::Core::Monitoring::Event]
+      # @note We can run the `#assignment_lost?` on each events poll because they happen once every
+      #   5 seconds during processing plus prior to each messages poll. It takes
+      #   0.6 microseconds per call.
+      def on_client_events_poll(event)
+        client = event[:caller]
+
+        # Only clear assignments if they were actually lost
+        return unless client.assignment_lost?
+
+        # Cleaning happens the same way as with the consumer reset
+        on_client_reset(event)
+      end
+
       # Removes partitions from the current assignments hash
       #
       # @param event [Karafka::Core::Monitoring::Event]

data/lib/karafka/instrumentation/monitor.rb

@@ -15,7 +15,7 @@ module Karafka
         notifications_bus = ::Karafka::Instrumentation::Notifications.new,
         namespace = nil
       )
-        super(notifications_bus, namespace)
+        super
       end
     end
   end

data/lib/karafka/instrumentation/notifications.rb

@@ -34,6 +34,7 @@ module Karafka
         client.pause
         client.resume
         client.reset
+        client.events_poll
 
         connection.listener.before_fetch_loop
         connection.listener.fetch_loop

data/lib/karafka/instrumentation/vendors/appsignal/base.rb

@@ -18,10 +18,9 @@ module Karafka
             setup(&block) if block
           end
 
-          # @param block [Proc] configuration block
           # @note We define this alias to be consistent with `Karafka#setup`
-          def setup(&block)
-            configure(&block)
+          def setup(&)
+            configure(&)
           end
         end
       end

data/lib/karafka/instrumentation/vendors/datadog/logger_listener.rb

@@ -38,10 +38,9 @@ module Karafka
             @job_types_cache = {}
           end
 
-          # @param block [Proc] configuration block
           # @note We define this alias to be consistent with `WaterDrop#setup`
-          def setup(&block)
-            configure(&block)
+          def setup(&)
+            configure(&)
           end
 
           # Prints info about the fact that a given job has started

data/lib/karafka/instrumentation/vendors/datadog/metrics_listener.rb

@@ -14,8 +14,9 @@ module Karafka
           include ::Karafka::Core::Configurable
           extend Forwardable
 
-          def_delegators :config, :client, :rd_kafka_metrics, :namespace,
-                         :default_tags, :distribution_mode
+          def_delegators(
+            :config, :client, :rd_kafka_metrics, :namespace, :default_tags, :distribution_mode
+          )
 
           # Value object for storing a single rdkafka metric publishing details
           RdKafkaMetric = Struct.new(:type, :scope, :name, :key_location)
@@ -69,10 +70,9 @@ module Karafka
             setup(&block) if block
           end
 
-          # @param block [Proc] configuration block
           # @note We define this alias to be consistent with `WaterDrop#setup`
-          def setup(&block)
-            configure(&block)
+          def setup(&)
+            configure(&)
           end
 
           # Hooks up to Karafka instrumentation for emitted statistics
@@ -198,18 +198,17 @@ module Karafka
 
           # Selects the histogram mode configured and uses it to report to DD client
           # @param key [String] non-namespaced key
-          # @param args [Array] extra arguments to pass to the client
-          def histogram(key, *args)
+          def histogram(key, *)
             case distribution_mode
             when :histogram
               client.histogram(
                 namespaced_metric(key),
-                *args
+                *
               )
             when :distribution
               client.distribution(
                 namespaced_metric(key),
-                *args
+                *
               )
             else
               raise(

data/lib/karafka/instrumentation/vendors/kubernetes/liveness_listener.rb

@@ -166,9 +166,8 @@ module Karafka
           end
 
           # Wraps the logic with a mutex
-          # @param block [Proc] code we want to run in mutex
-          def synchronize(&block)
-            @mutex.synchronize(&block)
+          def synchronize(&)
+            @mutex.synchronize(&)
           end
 
           # @return [Integer] object id of the current thread

data/lib/karafka/messages/builders/batch_metadata.rb

@@ -51,7 +51,7 @@ module Karafka
             return now unless last_message
 
             timestamp = last_message.timestamp
-            timestamp > now ? now : timestamp
+            [timestamp, now].min
           end
         end
       end

data/lib/karafka/messages/builders/message.rb

@@ -28,7 +28,7 @@ module Karafka
 
             # And nullify it in the kafka message. This can save a lot of memory when used with
             # the Pro Cleaner API
-            kafka_message.instance_variable_set('@payload', nil)
+            kafka_message.instance_variable_set(:@payload, nil)
 
             # Karafka messages cannot be frozen because of the lazy deserialization feature
             message = Karafka::Messages::Message.new(payload, metadata)

data/lib/karafka/messages/messages.rb

@@ -16,10 +16,9 @@ module Karafka
         @metadata = metadata
       end
 
-      # @param block [Proc] block we want to execute per each message
       # @note Invocation of this method will not cause loading and deserializing of messages.
-      def each(&block)
-        @messages_array.each(&block)
+      def each(&)
+        @messages_array.each(&)
       end
 
       # Runs deserialization of all the messages and returns them

data/lib/karafka/patches/rdkafka/bindings.rb

@@ -93,29 +93,29 @@ end
 
 # We need to replace the original callback with ours.
 # At the moment there is no API in rdkafka-ruby to do so
-::Rdkafka::Bindings.send(
+Rdkafka::Bindings.send(
   :remove_const,
   'RebalanceCallback'
 )
 
-::Rdkafka::Bindings.const_set(
-  'RebalanceCallback',
+Rdkafka::Bindings.const_set(
+  :RebalanceCallback,
   Karafka::Patches::Rdkafka::Bindings::RebalanceCallback
 )
 
-::Rdkafka::Bindings.attach_function(
+Rdkafka::Bindings.attach_function(
   :rd_kafka_rebalance_protocol,
   %i[pointer],
   :string
 )
 
-::Rdkafka::Bindings.attach_function(
+Rdkafka::Bindings.attach_function(
   :rd_kafka_incremental_assign,
   %i[pointer pointer],
   :string
 )
 
-::Rdkafka::Bindings.attach_function(
+Rdkafka::Bindings.attach_function(
   :rd_kafka_incremental_unassign,
   %i[pointer pointer],
   :string

data/lib/karafka/patches/rdkafka/opaque.rb

@@ -31,6 +31,6 @@ module Karafka
   end
 end
 
-::Rdkafka::Opaque.include(
+Rdkafka::Opaque.include(
   Karafka::Patches::Rdkafka::Opaque
 )