karafka 2.1.9 → 2.1.11

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 65296040c91ec5646620f047567c9229b7d9044d41d37e690f9f6685d1199e36
-  data.tar.gz: dedea921e5bc3b9b985861d9b6bcf6c5b5de1a148ef2c1aa72b0d96de44fb623
+  metadata.gz: 8d1e3000ced82b96afc7c4588d20170e0908d3b41305eead2e5e0a850c9a70c4
+  data.tar.gz: 1b16fc0a06f1bfd9bdb1207dd1faac04d3a7f956ccb42edd45ada76fed6dcf09
 SHA512:
-  metadata.gz: 781e45af707a9818e9166d6c37d8be214e9334703ee598f4ce8924c99ec71283cc4364de6662fbd1bb7454600b94b71e2e32750b58630e3631235e5bf593c3f9
-  data.tar.gz: b86b74c4d7909d504e0b60ae536360412622451a7a9868034dca63fbc3c5e962ba122967aeb8a1491196ce6b879b25eef11497798d861fcf85d3acca40fc8163
+  metadata.gz: 7e018450fae0ad666bff80ca9875750303946746c20e527200ee569a1a182c3d81bb71b64a0b2bd94b4a35c8cf204326b26752b1d5c0425a1f544ff9b4572323
+  data.tar.gz: 6025b3583179592313540231dd9d00354c30d8af3d49f482fc6c7dbd96359dbc7bbccee8cfa4cf1d14a2f5dfb3a9999b141317c782ec33af7503c4a8e26c616c

data/CHANGELOG.md

@@ -1,5 +1,21 @@
 # Karafka framework changelog
 
+## 2.1.11 (2023-08-23)
+- [Enhancement] Expand the error handling for offset related queries with timeout error retries.
+- [Enhancement] Allow for connection proxy timeouts configuration.
+
+## 2.1.10 (2023-08-21)
+- [Enhancement] Introduce `connection.client.rebalance_callback` event for instrumentation of rebalances.
+- [Refactor] Introduce low level commands proxy to handle deviation in how we want to run certain commands and how rdkafka-ruby runs that by design.
+- [Fix] Do not report lags in the DD listener for cases where the assignment is not workable.
+- [Fix] Do not report negative lags in the DD listener.
+- [Fix] Extremely fast shutdown after boot in specs can cause process not to stop.
+- [Fix] Disable `allow.auto.create.topics` for admin by default to prevent accidental topics creation on topics metadata lookups.
+- [Fix] Improve the `query_watermark_offsets` operations by increasing too low timeout.
+- [Fix] Increase `TplBuilder` timeouts to compensate for remote clusters.
+- [Fix] Always try to unsubscribe short-lived consumers used throughout the system, especially in the admin APIs.
+- [Fix] Add missing `connection.client.poll.error` error type reference.
+
 ## 2.1.9 (2023-08-06)
 - **[Feature]** Introduce ability to customize pause strategy on a per topic basis (Pro).
 - [Improvement] Disable the extensive messages logging in the default `karafka.rb` template.

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    karafka (2.1.9)
+    karafka (2.1.11)
       karafka-core (>= 2.1.1, < 2.2.0)
       thor (>= 0.20)
       waterdrop (>= 2.6.6, < 3.0.0)
@@ -10,10 +10,10 @@ PATH
 GEM
   remote: https://rubygems.org/
   specs:
-    activejob (7.0.6)
-      activesupport (= 7.0.6)
+    activejob (7.0.7)
+      activesupport (= 7.0.7)
       globalid (>= 0.3.6)
-    activesupport (7.0.6)
+    activesupport (7.0.7)
       concurrent-ruby (~> 1.0, >= 1.0.2)
       i18n (>= 1.6, < 2)
       minitest (>= 5.1)
@@ -33,7 +33,7 @@ GEM
     karafka-core (2.1.1)
       concurrent-ruby (>= 1.1)
       karafka-rdkafka (>= 0.13.1, < 0.14.0)
-    karafka-rdkafka (0.13.3)
+    karafka-rdkafka (0.13.4)
       ffi (~> 1.15)
       mini_portile2 (~> 2.6)
       rake (> 12)
@@ -44,10 +44,10 @@ GEM
       roda (~> 3.68, >= 3.68)
       tilt (~> 2.0)
     mini_portile2 (2.8.4)
-    minitest (5.18.1)
+    minitest (5.19.0)
     rack (3.0.8)
     rake (13.0.6)
-    roda (3.70.0)
+    roda (3.71.0)
       rack
     rspec (3.12.0)
       rspec-core (~> 3.12.0)
@@ -75,7 +75,7 @@ GEM
     waterdrop (2.6.6)
       karafka-core (>= 2.1.1, < 3.0.0)
       zeitwerk (~> 2.3)
-    zeitwerk (2.6.8)
+    zeitwerk (2.6.11)
 
 PLATFORMS
   x86_64-linux

data/certs/cert_chain.pem

@@ -1,26 +1,26 @@
 -----BEGIN CERTIFICATE-----
 MIIEcDCCAtigAwIBAgIBATANBgkqhkiG9w0BAQsFADA/MRAwDgYDVQQDDAdjb250
 YWN0MRcwFQYKCZImiZPyLGQBGRYHa2FyYWZrYTESMBAGCgmSJomT8ixkARkWAmlv
-MB4XDTIyMDgxOTE3MjEzN1oXDTIzMDgxOTE3MjEzN1owPzEQMA4GA1UEAwwHY29u
+MB4XDTIzMDgyMTA3MjU1NFoXDTI0MDgyMDA3MjU1NFowPzEQMA4GA1UEAwwHY29u
 dGFjdDEXMBUGCgmSJomT8ixkARkWB2thcmFma2ExEjAQBgoJkiaJk/IsZAEZFgJp
-bzCCAaIwDQYJKoZIhvcNAQEBBQADggGPADCCAYoCggGBAODzeO3L6lxdATzMHKNW
-jFA/GGunoPuylO/BMzy8RiQHh7VIvysAKs0tHhTx3g2D0STDpF+hcQcPELFikiT2
-F+1wOHj/SsrK7VKqfA8+gq04hKc5sQoX2Egf9k3V0YJ3eZ6R/koHkQ8A0TVt0w6F
-ZQckoV4MqnEAx0g/FZN3mnHTlJ3VFLSBqJEIe+S6FZMl92mSv+hTrlUG8VaYxSfN
-lTCvnKk284F6QZq5XIENLRmcDd/3aPBLnLwNnyMyhB+6gK8cUO+CFlDO5tjo/aBA
-rUnl++wGG0JooF1ed0v+evOn9KoMBG6rHewcf79qJbVOscbD8qSAmo+sCXtcFryr
-KRMTB8gNbowJkFRJDEe8tfRy11u1fYzFg/qNO82FJd62rKAw2wN0C29yCeQOPRb1
-Cw9Y4ZwK9VFNEcV9L+3pHTHn2XfuZHtDaG198VweiF6raFO4yiEYccodH/USP0L5
-cbcCFtmu/4HDSxL1ByQXO84A0ybJuk3/+aPUSXe9C9U8fwIDAQABo3cwdTAJBgNV
-HRMEAjAAMAsGA1UdDwQEAwIEsDAdBgNVHQ4EFgQUSlcEakb7gfn/5E2WY6z73BF/
-iZkwHQYDVR0RBBYwFIESY29udGFjdEBrYXJhZmthLmlvMB0GA1UdEgQWMBSBEmNv
-bnRhY3RAa2FyYWZrYS5pbzANBgkqhkiG9w0BAQsFAAOCAYEA1aS+E7RXJ1w9g9mJ
-G0NzFxe64OEuENosNlvYQCbRKGCXAU1qqelYkBQHseRgRKxLICrnypRo9IEobyHa
-vDnJ4r7Tsb34dleqQW2zY/obG+cia3Ym2JsegXWF7dDOzCXJ4FN8MFoT2jHlqLLw
-yrap0YO5zx0GSQ0Dwy8h2n2v2vanMEeCx7iNm3ERgR5WuN5sjzWoz2A/JLEEcK0C
-EnAGKCWAd1fuG8IemDjT1edsd5FyYR4bIX0m+99oDuFZyPiiIbalmyYiSBBp59Yb
-Q0P8zeBi4OfwCZNcxqz0KONmw9JLNv6DgyEAH5xe/4JzhMEgvIRiPj0pHfA7oqQF
-KUNqvD1KlxbEC+bZfE5IZhnqYLdld/Ksqd22FI1RBhiS1Ejfsj99LVIm9cBuZEY2
-Qf04B9ceLUaC4fPVEz10FyobjaFoY4i32xRto3XnrzeAgfEe4swLq8bQsR3w/EF3
-MGU0FeSV2Yj7Xc2x/7BzLK8xQn5l7Yy75iPF+KP3vVmDHnNl
+bzCCAaIwDQYJKoZIhvcNAQEBBQADggGPADCCAYoCggGBAOuZpyQKEwsTG9plLat7
+8bUaNuNBEnouTsNMr6X+XTgvyrAxTuocdsyP1sNCjdS1B8RiiDH1/Nt9qpvlBWon
+sdJ1SYhaWNVfqiYStTDnCx3PRMmHRdD4KqUWKpN6VpZ1O/Zu+9Mw0COmvXgZuuO9
+wMSJkXRo6dTCfMedLAIxjMeBIxtoLR2e6Jm6MR8+8WYYVWrO9kSOOt5eKQLBY7aK
+b/Dc40EcJKPg3Z30Pia1M9ZyRlb6SOj6SKpHRqc7vbVQxjEw6Jjal1lZ49m3YZMd
+ArMAs9lQZNdSw5/UX6HWWURLowg6k10RnhTUtYyzO9BFev0JFJftHnmuk8vtb+SD
+5VPmjFXg2VOcw0B7FtG75Vackk8QKfgVe3nSPhVpew2CSPlbJzH80wChbr19+e3+
+YGr1tOiaJrL6c+PNmb0F31NXMKpj/r+n15HwlTMRxQrzFcgjBlxf2XFGnPQXHhBm
+kp1OFnEq4GG9sON4glRldkwzi/f/fGcZmo5fm3d+0ZdNgwIDAQABo3cwdTAJBgNV
+HRMEAjAAMAsGA1UdDwQEAwIEsDAdBgNVHQ4EFgQUPVH5+dLA80A1kJ2Uz5iGwfOa
+1+swHQYDVR0RBBYwFIESY29udGFjdEBrYXJhZmthLmlvMB0GA1UdEgQWMBSBEmNv
+bnRhY3RAa2FyYWZrYS5pbzANBgkqhkiG9w0BAQsFAAOCAYEAnpa0jcN7JzREHMTQ
+bfZ+xcvlrzuROMY6A3zIZmQgbnoZZNuX4cMRrT1p1HuwXpxdpHPw7dDjYqWw3+1h
+3mXLeMuk7amjQpYoSWU/OIZMhIsARra22UN8qkkUlUj3AwTaChVKN/bPJOM2DzfU
+kz9vUgLeYYFfQbZqeI6SsM7ltilRV4W8D9yNUQQvOxCFxtLOetJ00fC/E7zMUzbK
+IBwYFQYsbI6XQzgAIPW6nGSYKgRhkfpmquXSNKZRIQ4V6bFrufa+DzD0bt2ZA3ah
+fMmJguyb5L2Gf1zpDXzFSPMG7YQFLzwYz1zZZvOU7/UCpQsHpID/YxqDp4+Dgb+Y
+qma0whX8UG/gXFV2pYWpYOfpatvahwi+A1TwPQsuZwkkhi1OyF1At3RY+hjSXyav
+AnG1dJU+yL2BK7vaVytLTstJME5mepSZ46qqIJXMuWob/YPDmVaBF39TDSG9e34s
+msG3BiCqgOgHAnL23+CN3Rt8MsuRfEtoTKpJVcCfoEoNHOkc
 -----END CERTIFICATE-----

data/config/locales/errors.yml

@@ -26,6 +26,12 @@ en:
       internal.process_format: needs to be present
       internal.routing.builder_format: needs to be present
       internal.routing.subscription_groups_builder_format: needs to be present
+      internal.connection.proxy.query_watermark_offsets.timeout_format: needs to be an integer bigger than 0
+      internal.connection.proxy.query_watermark_offsets.max_attempts_format: needs to be an integer bigger than 0
+      internal.connection.proxy.query_watermark_offsets.wait_time_format: needs to be an integer bigger than 0
+      internal.connection.proxy.offsets_for_times.timeout_format: needs to be an integer bigger than 0
+      internal.connection.proxy.offsets_for_times.max_attempts_format: needs to be an integer bigger than 0
+      internal.connection.proxy.offsets_for_times.wait_time_format: needs to be an integer bigger than 0
       key_must_be_a_symbol: All keys under the kafka settings scope need to be symbols
       max_timeout_vs_pause_max_timeout: pause_timeout must be less or equal to pause_max_timeout
       shutdown_timeout_vs_max_wait_time: shutdown_timeout must be more than max_wait_time

data/lib/karafka/admin.rb

@@ -13,9 +13,6 @@ module Karafka
     # retry after checking that the operation was finished or failed using external factor.
     MAX_WAIT_TIMEOUT = 1
 
-    # Max time for a TPL request. We increase it to compensate for remote clusters latency
-    TPL_REQUEST_TIMEOUT = 2_000
-
     # How many times should be try. 1 x 60 => 60 seconds wait in total
     MAX_ATTEMPTS = 60
 
@@ -29,11 +26,12 @@ module Karafka
       'fetch.message.max.bytes': 5 * 1_048_576,
       # Do not commit offset automatically, this prevents offset tracking for operations involving
       # a consumer instance
-      'enable.auto.commit': false
+      'enable.auto.commit': false,
+      # Make sure that topic metadata lookups do not create topics accidentally
+      'allow.auto.create.topics': false
     }.freeze
 
-    private_constant :CONFIG_DEFAULTS, :MAX_WAIT_TIMEOUT, :TPL_REQUEST_TIMEOUT,
-                     :MAX_ATTEMPTS
+    private_constant :CONFIG_DEFAULTS, :MAX_WAIT_TIMEOUT, :MAX_ATTEMPTS
 
     class << self
       # Allows us to read messages from the topic
@@ -184,10 +182,24 @@ module Karafka
       # This API can be used in other pieces of code and allows for low-level consumer usage
       #
       # @param settings [Hash] extra settings to customize consumer
+      #
+      # @note We always ship and yield a proxied consumer because admin API performance is not
+      #   that relevant. That is, there are no high frequency calls that would have to be delegated
       def with_consumer(settings = {})
         consumer = config(:consumer, settings).consumer
-        yield(consumer)
+        proxy = ::Karafka::Connection::Proxy.new(consumer)
+        yield(proxy)
       ensure
+        # Always unsubscribe consumer just to be sure, that no metadata requests are running
+        # when we close the consumer. This in theory should prevent from some race-conditions
+        # that originate from librdkafka
+        begin
+          consumer&.unsubscribe
+        # Ignore any errors and continue to close consumer despite them
+        rescue Rdkafka::RdkafkaError
+          nil
+        end
+
         consumer&.close
       end
 
@@ -261,7 +273,7 @@ module Karafka
             name, partition => offset
           )
 
-          real_offsets = consumer.offsets_for_times(tpl, TPL_REQUEST_TIMEOUT)
+          real_offsets = consumer.offsets_for_times(tpl)
           detected_offset = real_offsets.to_h.dig(name, partition)
 
           detected_offset&.offset || raise(Errors::InvalidTimeBasedOffsetError)

data/lib/karafka/app.rb

@@ -53,6 +53,12 @@ module Karafka
         RUBY
       end
 
+      # @return [Boolean] true if we should be done in general with processing anything
+      # @note It is a meta status from the status object
+      def done?
+        App.config.internal.status.done?
+      end
+
       # Methods that should be delegated to Karafka module
       %i[
         root

data/lib/karafka/base_consumer.rb

@@ -70,9 +70,9 @@ module Karafka
 
     # Executes the default consumer flow.
     #
-    # @return [Boolean] true if there was no exception, otherwise false.
-    #
     # @private
+    #
+    # @return [Boolean] true if there was no exception, otherwise false.
     # @note We keep the seek offset tracking, and use it to compensate for async offset flushing
     #   that may not yet kick in when error occurs. That way we pause always on the last processed
     #   message.

data/lib/karafka/connection/client.rb

@@ -20,9 +20,6 @@ module Karafka
       # How many times should we retry polling in case of a failure
       MAX_POLL_RETRIES = 20
 
-      # Max time for a TPL request. We increase it to compensate for remote clusters latency
-      TPL_REQUEST_TIMEOUT = 2_000
-
       # 1 minute of max wait for the first rebalance before a forceful attempt
       # This applies only to a case when a short-lived Karafka instance with a client would be
       # closed before first rebalance. Mitigates a librdkafka bug.
@@ -32,8 +29,7 @@ module Karafka
       # potential race conditions and other issues
       SHUTDOWN_MUTEX = Mutex.new
 
-      private_constant :MAX_POLL_RETRIES, :SHUTDOWN_MUTEX, :TPL_REQUEST_TIMEOUT,
-                       :COOPERATIVE_STICKY_MAX_WAIT
+      private_constant :MAX_POLL_RETRIES, :SHUTDOWN_MUTEX, :COOPERATIVE_STICKY_MAX_WAIT
 
       # Creates a new consumer instance.
       #
@@ -350,10 +346,12 @@ module Karafka
             message.partition => message.offset
           )
 
+          proxy = Proxy.new(@kafka)
+
           # Now we can overwrite the seek message offset with our resolved offset and we can
           # then seek to the appropriate message
           # We set the timeout to 2_000 to make sure that remote clusters handle this well
-          real_offsets = @kafka.offsets_for_times(tpl, TPL_REQUEST_TIMEOUT)
+          real_offsets = proxy.offsets_for_times(tpl)
           detected_partition = real_offsets.to_h.dig(message.topic, message.partition)
 
           # There always needs to be an offset. In case we seek into the future, where there
@@ -387,6 +385,21 @@ module Karafka
         end
       end
 
+      # Unsubscribes from all the subscriptions
+      # @note This is a private API to be used only on shutdown
+      # @note We do not re-raise since this is supposed to be only used on close and can be safely
+      #   ignored. We do however want to instrument on it
+      def unsubscribe
+        @kafka.unsubscribe
+      rescue ::Rdkafka::RdkafkaError => e
+        Karafka.monitor.instrument(
+          'error.occurred',
+          caller: self,
+          error: e,
+          type: 'connection.client.unsubscribe.error'
+        )
+      end
+
       # @param topic [String]
       # @param partition [Integer]
       # @return [Rdkafka::Consumer::TopicPartitionList]

data/lib/karafka/connection/listener.rb

@@ -85,7 +85,7 @@ module Karafka
       #   propagate this far.
       def fetch_loop
         # Run the main loop as long as we are not stopping or moving into quiet mode
-        until Karafka::App.stopping? || Karafka::App.quieting? || Karafka::App.quiet?
+        until Karafka::App.done?
           Karafka.monitor.instrument(
             'connection.listener.fetch_loop',
             caller: self,

data/lib/karafka/connection/proxy.rb

@@ -0,0 +1,98 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Connection
+    # Usually it is ok to use the `Rdkafka::Consumer` directly because we need 1:1 its
+    # functionality. There are however cases where we want to have extra recoveries or other
+    # handling of errors and settings. This is where this module comes in handy.
+    #
+    # We do not want to wrap and delegate all via a proxy object for performance reasons, but we
+    # do still want to be able to alter some functionalities. This wrapper helps us do it when
+    # it would be needed
+    class Proxy < SimpleDelegator
+      # Errors on which we want to retry
+      RETRYABLE_ERRORS = %i[
+        all_brokers_down
+        timed_out
+      ].freeze
+
+      private_constant :RETRYABLE_ERRORS
+
+      attr_accessor :wrapped
+
+      alias __getobj__ wrapped
+
+      # @param obj [Rdkafka::Consumer, Proxy] rdkafka consumer or consumer wrapped with proxy
+      def initialize(obj)
+        super
+        # Do not allow for wrapping proxy with a proxy. This will prevent a case where we might
+        # wrap an already wrapped object with another proxy level. Simplifies passing consumers
+        # and makes it safe to wrap without type checking
+        @wrapped = obj.is_a?(self.class) ? obj.wrapped : obj
+        @config = ::Karafka::App.config.internal.connection.proxy
+      end
+
+      # Proxies the `#query_watermark_offsets` with extra recovery from timeout problems.
+      # We impose our own custom timeout to make sure, that high-latency clusters and overloaded
+      # clusters can handle our requests.
+      #
+      # @param topic [String] topic name
+      # @param partition [Partition]
+      # @return [Array<Integer, Integer>] watermark offsets
+      def query_watermark_offsets(topic, partition)
+        l_config = @config.query_watermark_offsets
+
+        with_broker_errors_retry(
+          # required to be in seconds, not ms
+          wait_time: l_config.wait_time / 1_000.to_f,
+          max_attempts: l_config.max_attempts
+        ) do
+          @wrapped.query_watermark_offsets(topic, partition, l_config.timeout)
+        end
+      end
+
+      # Similar to `#query_watermark_offsets`, this method can be sensitive to latency. We handle
+      # this the same way
+      #
+      # @param tpl [Rdkafka::Consumer::TopicPartitionList] tpl to get time offsets
+      # @return [Rdkafka::Consumer::TopicPartitionList] tpl with time offsets
+      def offsets_for_times(tpl)
+        l_config = @config.offsets_for_times
+
+        with_broker_errors_retry(
+          # required to be in seconds, not ms
+          wait_time: l_config.wait_time / 1_000.to_f,
+          max_attempts: l_config.max_attempts
+        ) do
+          @wrapped.offsets_for_times(tpl, l_config.timeout)
+        end
+      end
+
+      private
+
+      # Runs expected block of code with few retries on all_brokers_down
+      # librdkafka can return `all_brokers_down` for scenarios when broker is overloaded or not
+      # reachable due to latency.
+      # @param max_attempts [Integer] how many attempts (not retries) should we take before failing
+      #   completely.
+      # @param wait_time [Integer, Float] how many seconds should we wait. It uses `#sleep` of Ruby
+      #   so it needs time in seconds.
+      def with_broker_errors_retry(max_attempts:, wait_time: 1)
+        attempt ||= 0
+        attempt += 1
+
+        yield
+      rescue Rdkafka::RdkafkaError => e
+        raise unless RETRYABLE_ERRORS.include?(e.code)
+
+        if attempt <= max_attempts
+          sleep(wait_time)
+
+          retry
+        end
+
+        raise
+      end
+    end
+  end
+end

data/lib/karafka/contracts/config.rb

@@ -39,6 +39,22 @@ module Karafka
         required(:status) { |val| !val.nil? }
         required(:process) { |val| !val.nil? }
 
+        nested(:connection) do
+          nested(:proxy) do
+            nested(:query_watermark_offsets) do
+              required(:timeout) { |val| val.is_a?(Integer) && val.positive? }
+              required(:max_attempts) { |val| val.is_a?(Integer) && val.positive? }
+              required(:wait_time) { |val| val.is_a?(Integer) && val.positive? }
+            end
+
+            nested(:offsets_for_times) do
+              required(:timeout) { |val| val.is_a?(Integer) && val.positive? }
+              required(:max_attempts) { |val| val.is_a?(Integer) && val.positive? }
+              required(:wait_time) { |val| val.is_a?(Integer) && val.positive? }
+            end
+          end
+        end
+
         nested(:routing) do
           required(:builder) { |val| !val.nil? }
           required(:subscription_groups_builder) { |val| !val.nil? }

data/lib/karafka/instrumentation/logger_listener.rb

@@ -280,6 +280,9 @@ module Karafka
         when 'connection.client.rebalance_callback.error'
           error "Rebalance callback error occurred: #{error}"
           error details
+        when 'connection.client.unsubscribe.error'
+          error "Client unsubscribe error occurred: #{error}"
+          error details
         else
           # This should never happen. Please contact the maintainers
           raise Errors::UnsupportedCaseError, event

data/lib/karafka/instrumentation/notifications.rb

@@ -35,6 +35,10 @@ module Karafka
         connection.listener.fetch_loop
         connection.listener.fetch_loop.received
 
+        connection.client.rebalance_callback
+        connection.client.poll.error
+        connection.client.unsubscribe.error
+
         consumer.consume
         consumer.consumed
         consumer.consuming.pause

data/lib/karafka/instrumentation/vendors/datadog/metrics_listener.rb

@@ -220,6 +220,11 @@ module Karafka
                   next if partition_name == '-1'
                   # Skip until lag info is available
                   next if partition_statistics['consumer_lag'] == -1
+                  next if partition_statistics['consumer_lag_stored'] == -1
+
+                  # Skip if we do not own the fetch assignment
+                  next if partition_statistics['fetch_state'] == 'stopped'
+                  next if partition_statistics['fetch_state'] == 'none'
 
                   public_send(
                     metric.type,

data/lib/karafka/patches/rdkafka/bindings.rb

@@ -1,7 +1,9 @@
 # frozen_string_literal: true
 
 module Karafka
+  # Namespace for third-party libraries patches
   module Patches
+    # Rdkafka patches specific to Karafka
     module Rdkafka
       # Binding patches that slightly change how rdkafka operates in certain places
       module Bindings
@@ -51,11 +53,18 @@ module Karafka
           # @param opaque [Rdkafka::Opaque]
           # @param tpl [Rdkafka::Consumer::TopicPartitionList]
           def trigger_callbacks(code, opaque, tpl)
-            case code
-            when RB::RD_KAFKA_RESP_ERR__ASSIGN_PARTITIONS
-              opaque.call_on_partitions_assigned(tpl)
-            when RB::RD_KAFKA_RESP_ERR__REVOKE_PARTITIONS
-              opaque.call_on_partitions_revoked(tpl)
+            Karafka.monitor.instrument(
+              'connection.client.rebalance_callback',
+              caller: self,
+              code: code,
+              tpl: tpl
+            ) do
+              case code
+              when RB::RD_KAFKA_RESP_ERR__ASSIGN_PARTITIONS
+                opaque.call_on_partitions_assigned(tpl)
+              when RB::RD_KAFKA_RESP_ERR__REVOKE_PARTITIONS
+                opaque.call_on_partitions_revoked(tpl)
+              end
             end
           rescue StandardError => e
             Karafka.monitor.instrument(

data/lib/karafka/pro/iterator/tpl_builder.rb

@@ -14,11 +14,6 @@
 module Karafka
   module Pro
     class Iterator
-      # Max time for a TPL request. We increase it to compensate for remote clusters latency
-      TPL_REQUEST_TIMEOUT = 2_000
-
-      private_constant :TPL_REQUEST_TIMEOUT
-
       # Because we have various formats in which we can provide the offsets, before we can
       # subscribe to them, there needs to be a bit of normalization.
       #
@@ -30,7 +25,7 @@ module Karafka
         # @param consumer [::Rdkafka::Consumer] consumer instance needed to talk with Kafka
         # @param expanded_topics [Hash] hash with expanded and normalized topics data
         def initialize(consumer, expanded_topics)
-          @consumer = consumer
+          @consumer = Connection::Proxy.new(consumer)
           @expanded_topics = expanded_topics
           @mapped_topics = Hash.new { |h, k| h[k] = {} }
         end
@@ -144,7 +139,7 @@ module Karafka
           # If there were no time-based, no need to query Kafka
           return if time_tpl.empty?
 
-          real_offsets = @consumer.offsets_for_times(time_tpl, TPL_REQUEST_TIMEOUT)
+          real_offsets = @consumer.offsets_for_times(time_tpl)
 
           real_offsets.to_h.each do |name, results|
             results.each do |result|

data/lib/karafka/setup/config.rb

@@ -126,6 +126,32 @@ module Karafka
           setting :activity_manager, default: Routing::ActivityManager.new
         end
 
+        # Namespace for internal connection related settings
+        setting :connection do
+          # Settings that are altered by our client proxy layer
+          setting :proxy do
+            # Watermark offsets request settings
+            setting :query_watermark_offsets do
+              # timeout for this request. For busy or remote clusters, this should be high enough
+              setting :timeout, default: 5_000
+              # How many times should we try to run this call before raising an error
+              setting :max_attempts, default: 3
+              # How long should we wait before next attempt in case of a failure
+              setting :wait_time, default: 1_000
+            end
+
+            # Offsets for times request settings
+            setting :offsets_for_times do
+              # timeout for this request. For busy or remote clusters, this should be high enough
+              setting :timeout, default: 5_000
+              # How many times should we try to run this call before raising an error
+              setting :max_attempts, default: 3
+              # How long should we wait before next attempt in case of a failure
+              setting :wait_time, default: 1_000
+            end
+          end
+        end
+
         setting :processing do
           # option scheduler [Object] scheduler we will be using
           setting :scheduler, default: Processing::Scheduler.new

data/lib/karafka/status.rb

@@ -62,5 +62,15 @@ module Karafka
         end
       end
     end
+
+    # @return [Boolean] true if we are in any of the status that would indicate we should no longer
+    #   process incoming data. It is a meta status built from others and not a separate state in
+    #   the sense of a state machine
+    def done?
+      # Short-track for the most common case not to invoke all others on normal execution
+      return false if running?
+
+      stopping? || stopped? || quieting? || quiet? || terminated?
+    end
   end
 end

data/lib/karafka/version.rb

@@ -3,5 +3,5 @@
 # Main module namespace
 module Karafka
   # Current Karafka version
-  VERSION = '2.1.9'
+  VERSION = '2.1.11'
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: karafka
 version: !ruby/object:Gem::Version
-  version: 2.1.9
+  version: 2.1.11
 platform: ruby
 authors:
 - Maciej Mensfeld
@@ -12,30 +12,30 @@ cert_chain:
   -----BEGIN CERTIFICATE-----
   MIIEcDCCAtigAwIBAgIBATANBgkqhkiG9w0BAQsFADA/MRAwDgYDVQQDDAdjb250
   YWN0MRcwFQYKCZImiZPyLGQBGRYHa2FyYWZrYTESMBAGCgmSJomT8ixkARkWAmlv
-  MB4XDTIyMDgxOTE3MjEzN1oXDTIzMDgxOTE3MjEzN1owPzEQMA4GA1UEAwwHY29u
+  MB4XDTIzMDgyMTA3MjU1NFoXDTI0MDgyMDA3MjU1NFowPzEQMA4GA1UEAwwHY29u
   dGFjdDEXMBUGCgmSJomT8ixkARkWB2thcmFma2ExEjAQBgoJkiaJk/IsZAEZFgJp
-  bzCCAaIwDQYJKoZIhvcNAQEBBQADggGPADCCAYoCggGBAODzeO3L6lxdATzMHKNW
-  jFA/GGunoPuylO/BMzy8RiQHh7VIvysAKs0tHhTx3g2D0STDpF+hcQcPELFikiT2
-  F+1wOHj/SsrK7VKqfA8+gq04hKc5sQoX2Egf9k3V0YJ3eZ6R/koHkQ8A0TVt0w6F
-  ZQckoV4MqnEAx0g/FZN3mnHTlJ3VFLSBqJEIe+S6FZMl92mSv+hTrlUG8VaYxSfN
-  lTCvnKk284F6QZq5XIENLRmcDd/3aPBLnLwNnyMyhB+6gK8cUO+CFlDO5tjo/aBA
-  rUnl++wGG0JooF1ed0v+evOn9KoMBG6rHewcf79qJbVOscbD8qSAmo+sCXtcFryr
-  KRMTB8gNbowJkFRJDEe8tfRy11u1fYzFg/qNO82FJd62rKAw2wN0C29yCeQOPRb1
-  Cw9Y4ZwK9VFNEcV9L+3pHTHn2XfuZHtDaG198VweiF6raFO4yiEYccodH/USP0L5
-  cbcCFtmu/4HDSxL1ByQXO84A0ybJuk3/+aPUSXe9C9U8fwIDAQABo3cwdTAJBgNV
-  HRMEAjAAMAsGA1UdDwQEAwIEsDAdBgNVHQ4EFgQUSlcEakb7gfn/5E2WY6z73BF/
-  iZkwHQYDVR0RBBYwFIESY29udGFjdEBrYXJhZmthLmlvMB0GA1UdEgQWMBSBEmNv
-  bnRhY3RAa2FyYWZrYS5pbzANBgkqhkiG9w0BAQsFAAOCAYEA1aS+E7RXJ1w9g9mJ
-  G0NzFxe64OEuENosNlvYQCbRKGCXAU1qqelYkBQHseRgRKxLICrnypRo9IEobyHa
-  vDnJ4r7Tsb34dleqQW2zY/obG+cia3Ym2JsegXWF7dDOzCXJ4FN8MFoT2jHlqLLw
-  yrap0YO5zx0GSQ0Dwy8h2n2v2vanMEeCx7iNm3ERgR5WuN5sjzWoz2A/JLEEcK0C
-  EnAGKCWAd1fuG8IemDjT1edsd5FyYR4bIX0m+99oDuFZyPiiIbalmyYiSBBp59Yb
-  Q0P8zeBi4OfwCZNcxqz0KONmw9JLNv6DgyEAH5xe/4JzhMEgvIRiPj0pHfA7oqQF
-  KUNqvD1KlxbEC+bZfE5IZhnqYLdld/Ksqd22FI1RBhiS1Ejfsj99LVIm9cBuZEY2
-  Qf04B9ceLUaC4fPVEz10FyobjaFoY4i32xRto3XnrzeAgfEe4swLq8bQsR3w/EF3
-  MGU0FeSV2Yj7Xc2x/7BzLK8xQn5l7Yy75iPF+KP3vVmDHnNl
+  bzCCAaIwDQYJKoZIhvcNAQEBBQADggGPADCCAYoCggGBAOuZpyQKEwsTG9plLat7
+  8bUaNuNBEnouTsNMr6X+XTgvyrAxTuocdsyP1sNCjdS1B8RiiDH1/Nt9qpvlBWon
+  sdJ1SYhaWNVfqiYStTDnCx3PRMmHRdD4KqUWKpN6VpZ1O/Zu+9Mw0COmvXgZuuO9
+  wMSJkXRo6dTCfMedLAIxjMeBIxtoLR2e6Jm6MR8+8WYYVWrO9kSOOt5eKQLBY7aK
+  b/Dc40EcJKPg3Z30Pia1M9ZyRlb6SOj6SKpHRqc7vbVQxjEw6Jjal1lZ49m3YZMd
+  ArMAs9lQZNdSw5/UX6HWWURLowg6k10RnhTUtYyzO9BFev0JFJftHnmuk8vtb+SD
+  5VPmjFXg2VOcw0B7FtG75Vackk8QKfgVe3nSPhVpew2CSPlbJzH80wChbr19+e3+
+  YGr1tOiaJrL6c+PNmb0F31NXMKpj/r+n15HwlTMRxQrzFcgjBlxf2XFGnPQXHhBm
+  kp1OFnEq4GG9sON4glRldkwzi/f/fGcZmo5fm3d+0ZdNgwIDAQABo3cwdTAJBgNV
+  HRMEAjAAMAsGA1UdDwQEAwIEsDAdBgNVHQ4EFgQUPVH5+dLA80A1kJ2Uz5iGwfOa
+  1+swHQYDVR0RBBYwFIESY29udGFjdEBrYXJhZmthLmlvMB0GA1UdEgQWMBSBEmNv
+  bnRhY3RAa2FyYWZrYS5pbzANBgkqhkiG9w0BAQsFAAOCAYEAnpa0jcN7JzREHMTQ
+  bfZ+xcvlrzuROMY6A3zIZmQgbnoZZNuX4cMRrT1p1HuwXpxdpHPw7dDjYqWw3+1h
+  3mXLeMuk7amjQpYoSWU/OIZMhIsARra22UN8qkkUlUj3AwTaChVKN/bPJOM2DzfU
+  kz9vUgLeYYFfQbZqeI6SsM7ltilRV4W8D9yNUQQvOxCFxtLOetJ00fC/E7zMUzbK
+  IBwYFQYsbI6XQzgAIPW6nGSYKgRhkfpmquXSNKZRIQ4V6bFrufa+DzD0bt2ZA3ah
+  fMmJguyb5L2Gf1zpDXzFSPMG7YQFLzwYz1zZZvOU7/UCpQsHpID/YxqDp4+Dgb+Y
+  qma0whX8UG/gXFV2pYWpYOfpatvahwi+A1TwPQsuZwkkhi1OyF1At3RY+hjSXyav
+  AnG1dJU+yL2BK7vaVytLTstJME5mepSZ46qqIJXMuWob/YPDmVaBF39TDSG9e34s
+  msG3BiCqgOgHAnL23+CN3Rt8MsuRfEtoTKpJVcCfoEoNHOkc
   -----END CERTIFICATE-----
-date: 2023-08-06 00:00:00.000000000 Z
+date: 2023-08-23 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: karafka-core
@@ -178,6 +178,7 @@ files:
 - lib/karafka/connection/listeners_batch.rb
 - lib/karafka/connection/messages_buffer.rb
 - lib/karafka/connection/pauses_manager.rb
+- lib/karafka/connection/proxy.rb
 - lib/karafka/connection/raw_messages_buffer.rb
 - lib/karafka/connection/rebalance_manager.rb
 - lib/karafka/contracts.rb