karafka 2.1.8 → 2.1.10

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 042f365fb134a24ae360d678590ce798014751c8a23fb267001c920a42aa5324
-  data.tar.gz: ba2950de557a5f6c775577ce392d60ee839184f50b7d9225969c684625c9ecd0
+  metadata.gz: 78db49b9cd20426b92ad1a3417b9bc2e79e9efad7f7d11e42873ffda75b850c2
+  data.tar.gz: 40b2a338c11ad9d6d07fdb0b82f422fe8f64417805db85a9bd111f9bce29039f
 SHA512:
-  metadata.gz: 30b5fcd92c348c50482cb84542380ad28b317d47e01efad2d2049cd3ba7872c5f66a7a4fde93d8bfa262d7fcb3745dedbdb89a4fd5e6cdf2288a43606ea2361d
-  data.tar.gz: d9c8ba95b2b71f46a3d35e2f5be634473a7aa9f45d9b5924e1019dcb53c7cea6aca8c594f9f4d3bf85a14176c3cb98a7167b4eb6e1f6f2192059d8040440e4a9
+  metadata.gz: a522abfbe63ebde9255032b5c6760016a014b8952f8c3f593c754479921408451ac0aeca632ba32d1fcefac78acd97a110e777949f6e70587d454febf5ad34f5
+  data.tar.gz: d12c97a4829bb0278041ce20e38f21c4894c77a7c9d4b3e4c1c222c502632fe03b0cce3d224f4b8c28ec3743c9c51e41c98a53310761687e92ce587445eae300

data/CHANGELOG.md

@@ -1,5 +1,22 @@
 # Karafka framework changelog
 
+## 2.1.10 (2023-08-21)
+- [Enhancement] Introduce `connection.client.rebalance_callback` event for instrumentation of rebalances.
+- [Refactor] Introduce low level commands proxy to handle deviation in how we want to run certain commands and how rdkafka-ruby runs that by design.
+- [Fix] Do not report lags in the DD listener for cases where the assignment is not workable.
+- [Fix] Do not report negative lags in the DD listener.
+- [Fix] Extremely fast shutdown after boot in specs can cause process not to stop.
+- [Fix] Disable `allow.auto.create.topics` for admin by default to prevent accidental topics creation on topics metadata lookups.
+- [Fix] Improve the `query_watermark_offsets` operations by increasing too low timeout.
+- [Fix] Increase `TplBuilder` timeouts to compensate for remote clusters.
+- [Fix] Always try to unsubscribe short-lived consumers used throughout the system, especially in the admin APIs.
+- [Fix] Add missing `connection.client.poll.error` error type reference.
+
+## 2.1.9 (2023-08-06)
+- **[Feature]** Introduce ability to customize pause strategy on a per topic basis (Pro).
+- [Improvement] Disable the extensive messages logging in the default `karafka.rb` template.
+- [Change] Require `waterdrop` `>= 2.6.6` due to extra `LoggerListener` API.
+
 ## 2.1.8 (2023-07-29)
 - [Improvement] Introduce `Karafka::BaseConsumer#used?` method to indicate, that at least one invocation of `#consume` took or will take place. This can be used as a replacement to the non-direct `messages.count` check for shutdown and revocation to ensure, that the consumption took place or is taking place (in case of running LRJ).
 - [Improvement] Make `messages#to_a` return copy of the underlying array to prevent scenarios, where the mutation impacts offset management.

data/Gemfile.lock

@@ -1,19 +1,19 @@
 PATH
   remote: .
   specs:
-    karafka (2.1.8)
+    karafka (2.1.10)
       karafka-core (>= 2.1.1, < 2.2.0)
       thor (>= 0.20)
-      waterdrop (>= 2.6.2, < 3.0.0)
+      waterdrop (>= 2.6.6, < 3.0.0)
       zeitwerk (~> 2.3)
 
 GEM
   remote: https://rubygems.org/
   specs:
-    activejob (7.0.6)
-      activesupport (= 7.0.6)
+    activejob (7.0.7)
+      activesupport (= 7.0.7)
       globalid (>= 0.3.6)
-    activesupport (7.0.6)
+    activesupport (7.0.7)
       concurrent-ruby (~> 1.0, >= 1.0.2)
       i18n (>= 1.6, < 2)
       minitest (>= 5.1)
@@ -44,10 +44,10 @@ GEM
       roda (~> 3.68, >= 3.68)
       tilt (~> 2.0)
     mini_portile2 (2.8.4)
-    minitest (5.18.1)
+    minitest (5.19.0)
     rack (3.0.8)
     rake (13.0.6)
-    roda (3.70.0)
+    roda (3.71.0)
       rack
     rspec (3.12.0)
       rspec-core (~> 3.12.0)
@@ -72,10 +72,10 @@ GEM
     tilt (2.2.0)
     tzinfo (2.0.6)
       concurrent-ruby (~> 1.0)
-    waterdrop (2.6.5)
+    waterdrop (2.6.6)
       karafka-core (>= 2.1.1, < 3.0.0)
       zeitwerk (~> 2.3)
-    zeitwerk (2.6.8)
+    zeitwerk (2.6.11)
 
 PLATFORMS
   x86_64-linux

data/certs/cert_chain.pem

@@ -1,26 +1,26 @@
 -----BEGIN CERTIFICATE-----
 MIIEcDCCAtigAwIBAgIBATANBgkqhkiG9w0BAQsFADA/MRAwDgYDVQQDDAdjb250
 YWN0MRcwFQYKCZImiZPyLGQBGRYHa2FyYWZrYTESMBAGCgmSJomT8ixkARkWAmlv
-MB4XDTIyMDgxOTE3MjEzN1oXDTIzMDgxOTE3MjEzN1owPzEQMA4GA1UEAwwHY29u
+MB4XDTIzMDgyMTA3MjU1NFoXDTI0MDgyMDA3MjU1NFowPzEQMA4GA1UEAwwHY29u
 dGFjdDEXMBUGCgmSJomT8ixkARkWB2thcmFma2ExEjAQBgoJkiaJk/IsZAEZFgJp
-bzCCAaIwDQYJKoZIhvcNAQEBBQADggGPADCCAYoCggGBAODzeO3L6lxdATzMHKNW
-jFA/GGunoPuylO/BMzy8RiQHh7VIvysAKs0tHhTx3g2D0STDpF+hcQcPELFikiT2
-F+1wOHj/SsrK7VKqfA8+gq04hKc5sQoX2Egf9k3V0YJ3eZ6R/koHkQ8A0TVt0w6F
-ZQckoV4MqnEAx0g/FZN3mnHTlJ3VFLSBqJEIe+S6FZMl92mSv+hTrlUG8VaYxSfN
-lTCvnKk284F6QZq5XIENLRmcDd/3aPBLnLwNnyMyhB+6gK8cUO+CFlDO5tjo/aBA
-rUnl++wGG0JooF1ed0v+evOn9KoMBG6rHewcf79qJbVOscbD8qSAmo+sCXtcFryr
-KRMTB8gNbowJkFRJDEe8tfRy11u1fYzFg/qNO82FJd62rKAw2wN0C29yCeQOPRb1
-Cw9Y4ZwK9VFNEcV9L+3pHTHn2XfuZHtDaG198VweiF6raFO4yiEYccodH/USP0L5
-cbcCFtmu/4HDSxL1ByQXO84A0ybJuk3/+aPUSXe9C9U8fwIDAQABo3cwdTAJBgNV
-HRMEAjAAMAsGA1UdDwQEAwIEsDAdBgNVHQ4EFgQUSlcEakb7gfn/5E2WY6z73BF/
-iZkwHQYDVR0RBBYwFIESY29udGFjdEBrYXJhZmthLmlvMB0GA1UdEgQWMBSBEmNv
-bnRhY3RAa2FyYWZrYS5pbzANBgkqhkiG9w0BAQsFAAOCAYEA1aS+E7RXJ1w9g9mJ
-G0NzFxe64OEuENosNlvYQCbRKGCXAU1qqelYkBQHseRgRKxLICrnypRo9IEobyHa
-vDnJ4r7Tsb34dleqQW2zY/obG+cia3Ym2JsegXWF7dDOzCXJ4FN8MFoT2jHlqLLw
-yrap0YO5zx0GSQ0Dwy8h2n2v2vanMEeCx7iNm3ERgR5WuN5sjzWoz2A/JLEEcK0C
-EnAGKCWAd1fuG8IemDjT1edsd5FyYR4bIX0m+99oDuFZyPiiIbalmyYiSBBp59Yb
-Q0P8zeBi4OfwCZNcxqz0KONmw9JLNv6DgyEAH5xe/4JzhMEgvIRiPj0pHfA7oqQF
-KUNqvD1KlxbEC+bZfE5IZhnqYLdld/Ksqd22FI1RBhiS1Ejfsj99LVIm9cBuZEY2
-Qf04B9ceLUaC4fPVEz10FyobjaFoY4i32xRto3XnrzeAgfEe4swLq8bQsR3w/EF3
-MGU0FeSV2Yj7Xc2x/7BzLK8xQn5l7Yy75iPF+KP3vVmDHnNl
+bzCCAaIwDQYJKoZIhvcNAQEBBQADggGPADCCAYoCggGBAOuZpyQKEwsTG9plLat7
+8bUaNuNBEnouTsNMr6X+XTgvyrAxTuocdsyP1sNCjdS1B8RiiDH1/Nt9qpvlBWon
+sdJ1SYhaWNVfqiYStTDnCx3PRMmHRdD4KqUWKpN6VpZ1O/Zu+9Mw0COmvXgZuuO9
+wMSJkXRo6dTCfMedLAIxjMeBIxtoLR2e6Jm6MR8+8WYYVWrO9kSOOt5eKQLBY7aK
+b/Dc40EcJKPg3Z30Pia1M9ZyRlb6SOj6SKpHRqc7vbVQxjEw6Jjal1lZ49m3YZMd
+ArMAs9lQZNdSw5/UX6HWWURLowg6k10RnhTUtYyzO9BFev0JFJftHnmuk8vtb+SD
+5VPmjFXg2VOcw0B7FtG75Vackk8QKfgVe3nSPhVpew2CSPlbJzH80wChbr19+e3+
+YGr1tOiaJrL6c+PNmb0F31NXMKpj/r+n15HwlTMRxQrzFcgjBlxf2XFGnPQXHhBm
+kp1OFnEq4GG9sON4glRldkwzi/f/fGcZmo5fm3d+0ZdNgwIDAQABo3cwdTAJBgNV
+HRMEAjAAMAsGA1UdDwQEAwIEsDAdBgNVHQ4EFgQUPVH5+dLA80A1kJ2Uz5iGwfOa
+1+swHQYDVR0RBBYwFIESY29udGFjdEBrYXJhZmthLmlvMB0GA1UdEgQWMBSBEmNv
+bnRhY3RAa2FyYWZrYS5pbzANBgkqhkiG9w0BAQsFAAOCAYEAnpa0jcN7JzREHMTQ
+bfZ+xcvlrzuROMY6A3zIZmQgbnoZZNuX4cMRrT1p1HuwXpxdpHPw7dDjYqWw3+1h
+3mXLeMuk7amjQpYoSWU/OIZMhIsARra22UN8qkkUlUj3AwTaChVKN/bPJOM2DzfU
+kz9vUgLeYYFfQbZqeI6SsM7ltilRV4W8D9yNUQQvOxCFxtLOetJ00fC/E7zMUzbK
+IBwYFQYsbI6XQzgAIPW6nGSYKgRhkfpmquXSNKZRIQ4V6bFrufa+DzD0bt2ZA3ah
+fMmJguyb5L2Gf1zpDXzFSPMG7YQFLzwYz1zZZvOU7/UCpQsHpID/YxqDp4+Dgb+Y
+qma0whX8UG/gXFV2pYWpYOfpatvahwi+A1TwPQsuZwkkhi1OyF1At3RY+hjSXyav
+AnG1dJU+yL2BK7vaVytLTstJME5mepSZ46qqIJXMuWob/YPDmVaBF39TDSG9e34s
+msG3BiCqgOgHAnL23+CN3Rt8MsuRfEtoTKpJVcCfoEoNHOkc
 -----END CERTIFICATE-----

data/config/locales/pro_errors.yml

@@ -23,6 +23,11 @@ en:
       delaying.delay_format: 'needs to be equal or more than 0 and an integer'
       delaying.active_format: 'needs to be boolean'
 
+      pause_timeout_format: needs to be an integer bigger than 0
+      pause_max_timeout_format: needs to be an integer bigger than 0
+      pause_with_exponential_backoff_format: needs to be either true or false
+      pause_timeout_max_timeout_vs_pause_max_timeout: pause_timeout must be less or equal to pause_max_timeout
+
     config:
       encryption.active_format: 'needs to be either true or false'
       encryption.public_key_invalid: 'is not a valid public RSA key'

data/karafka.gemspec

@@ -23,7 +23,7 @@ Gem::Specification.new do |spec|
 
   spec.add_dependency 'karafka-core', '>= 2.1.1', '< 2.2.0'
   spec.add_dependency 'thor', '>= 0.20'
-  spec.add_dependency 'waterdrop', '>= 2.6.2', '< 3.0.0'
+  spec.add_dependency 'waterdrop', '>= 2.6.6', '< 3.0.0'
   spec.add_dependency 'zeitwerk', '~> 2.3'
 
   if $PROGRAM_NAME.end_with?('gem')

data/lib/karafka/admin.rb

@@ -13,9 +13,6 @@ module Karafka
     # retry after checking that the operation was finished or failed using external factor.
     MAX_WAIT_TIMEOUT = 1
 
-    # Max time for a TPL request. We increase it to compensate for remote clusters latency
-    TPL_REQUEST_TIMEOUT = 2_000
-
     # How many times should be try. 1 x 60 => 60 seconds wait in total
     MAX_ATTEMPTS = 60
 
@@ -29,11 +26,12 @@ module Karafka
       'fetch.message.max.bytes': 5 * 1_048_576,
       # Do not commit offset automatically, this prevents offset tracking for operations involving
       # a consumer instance
-      'enable.auto.commit': false
+      'enable.auto.commit': false,
+      # Make sure that topic metadata lookups do not create topics accidentally
+      'allow.auto.create.topics': false
     }.freeze
 
-    private_constant :CONFIG_DEFAULTS, :MAX_WAIT_TIMEOUT, :TPL_REQUEST_TIMEOUT,
-                     :MAX_ATTEMPTS
+    private_constant :CONFIG_DEFAULTS, :MAX_WAIT_TIMEOUT, :MAX_ATTEMPTS
 
     class << self
       # Allows us to read messages from the topic
@@ -184,10 +182,24 @@ module Karafka
       # This API can be used in other pieces of code and allows for low-level consumer usage
       #
       # @param settings [Hash] extra settings to customize consumer
+      #
+      # @note We always ship and yield a proxied consumer because admin API performance is not
+      #   that relevant. That is, there are no high frequency calls that would have to be delegated
       def with_consumer(settings = {})
         consumer = config(:consumer, settings).consumer
-        yield(consumer)
+        proxy = ::Karafka::Connection::Proxy.new(consumer)
+        yield(proxy)
       ensure
+        # Always unsubscribe consumer just to be sure, that no metadata requests are running
+        # when we close the consumer. This in theory should prevent from some race-conditions
+        # that originate from librdkafka
+        begin
+          consumer&.unsubscribe
+        # Ignore any errors and continue to close consumer despite them
+        rescue Rdkafka::RdkafkaError
+          nil
+        end
+
         consumer&.close
       end
 
@@ -261,7 +273,7 @@ module Karafka
             name, partition => offset
           )
 
-          real_offsets = consumer.offsets_for_times(tpl, TPL_REQUEST_TIMEOUT)
+          real_offsets = consumer.offsets_for_times(tpl)
           detected_offset = real_offsets.to_h.dig(name, partition)
 
           detected_offset&.offset || raise(Errors::InvalidTimeBasedOffsetError)

data/lib/karafka/app.rb

@@ -53,6 +53,12 @@ module Karafka
         RUBY
       end
 
+      # @return [Boolean] true if we should be done in general with processing anything
+      # @note It is a meta status from the status object
+      def done?
+        App.config.internal.status.done?
+      end
+
       # Methods that should be delegated to Karafka module
       %i[
         root

data/lib/karafka/base_consumer.rb

@@ -70,9 +70,9 @@ module Karafka
 
     # Executes the default consumer flow.
     #
-    # @return [Boolean] true if there was no exception, otherwise false.
-    #
     # @private
+    #
+    # @return [Boolean] true if there was no exception, otherwise false.
     # @note We keep the seek offset tracking, and use it to compensate for async offset flushing
     #   that may not yet kick in when error occurs. That way we pause always on the last processed
     #   message.

data/lib/karafka/connection/client.rb

@@ -20,9 +20,6 @@ module Karafka
       # How many times should we retry polling in case of a failure
       MAX_POLL_RETRIES = 20
 
-      # Max time for a TPL request. We increase it to compensate for remote clusters latency
-      TPL_REQUEST_TIMEOUT = 2_000
-
       # 1 minute of max wait for the first rebalance before a forceful attempt
       # This applies only to a case when a short-lived Karafka instance with a client would be
       # closed before first rebalance. Mitigates a librdkafka bug.
@@ -32,8 +29,7 @@ module Karafka
       # potential race conditions and other issues
       SHUTDOWN_MUTEX = Mutex.new
 
-      private_constant :MAX_POLL_RETRIES, :SHUTDOWN_MUTEX, :TPL_REQUEST_TIMEOUT,
-                       :COOPERATIVE_STICKY_MAX_WAIT
+      private_constant :MAX_POLL_RETRIES, :SHUTDOWN_MUTEX, :COOPERATIVE_STICKY_MAX_WAIT
 
       # Creates a new consumer instance.
       #
@@ -350,10 +346,12 @@ module Karafka
             message.partition => message.offset
           )
 
+          proxy = Proxy.new(@kafka)
+
           # Now we can overwrite the seek message offset with our resolved offset and we can
           # then seek to the appropriate message
           # We set the timeout to 2_000 to make sure that remote clusters handle this well
-          real_offsets = @kafka.offsets_for_times(tpl, TPL_REQUEST_TIMEOUT)
+          real_offsets = proxy.offsets_for_times(tpl)
           detected_partition = real_offsets.to_h.dig(message.topic, message.partition)
 
           # There always needs to be an offset. In case we seek into the future, where there
@@ -387,6 +385,21 @@ module Karafka
         end
       end
 
+      # Unsubscribes from all the subscriptions
+      # @note This is a private API to be used only on shutdown
+      # @note We do not re-raise since this is supposed to be only used on close and can be safely
+      #   ignored. We do however want to instrument on it
+      def unsubscribe
+        @kafka.unsubscribe
+      rescue ::Rdkafka::RdkafkaError => e
+        Karafka.monitor.instrument(
+          'error.occurred',
+          caller: self,
+          error: e,
+          type: 'connection.client.unsubscribe.error'
+        )
+      end
+
       # @param topic [String]
       # @param partition [Integer]
       # @return [Rdkafka::Consumer::TopicPartitionList]

data/lib/karafka/connection/listener.rb

@@ -85,7 +85,7 @@ module Karafka
       #   propagate this far.
       def fetch_loop
         # Run the main loop as long as we are not stopping or moving into quiet mode
-        until Karafka::App.stopping? || Karafka::App.quieting? || Karafka::App.quiet?
+        until Karafka::App.done?
           Karafka.monitor.instrument(
             'connection.listener.fetch_loop',
             caller: self,
@@ -192,7 +192,7 @@ module Karafka
       # Resumes processing of partitions that were paused due to an error.
       def resume_paused_partitions
         @coordinators.resume do |topic, partition|
-          @client.resume(topic, partition)
+          @client.resume(topic.name, partition)
         end
       end
 

data/lib/karafka/connection/pauses_manager.rb

@@ -14,20 +14,20 @@ module Karafka
 
       # Creates or fetches pause tracker of a given topic partition.
       #
-      # @param topic [String] topic name
+      # @param topic [::Karafka::Routing::Topic] topic
       # @param partition [Integer] partition number
       # @return [Karafka::TimeTrackers::Pause] pause tracker instance
       def fetch(topic, partition)
         @pauses[topic][partition] ||= TimeTrackers::Pause.new(
-          timeout: Karafka::App.config.pause_timeout,
-          max_timeout: Karafka::App.config.pause_max_timeout,
-          exponential_backoff: Karafka::App.config.pause_with_exponential_backoff
+          timeout: topic.pause_timeout,
+          max_timeout: topic.pause_max_timeout,
+          exponential_backoff: topic.pause_with_exponential_backoff
         )
       end
 
       # Resumes processing of partitions for which pause time has ended.
       #
-      # @yieldparam [String] topic name
+      # @yieldparam [Karafka::Routing::Topic] topic
       # @yieldparam [Integer] partition number
       def resume
         @pauses.each do |topic, partitions|

data/lib/karafka/connection/proxy.rb

@@ -0,0 +1,92 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Connection
+    # Usually it is ok to use the `Rdkafka::Consumer` directly because we need 1:1 its
+    # functionality. There are however cases where we want to have extra recoveries or other
+    # handling of errors and settings. This is where this module comes in handy.
+    #
+    # We do not want to wrap and delegate all via a proxy object for performance reasons, but we
+    # do still want to be able to alter some functionalities. This wrapper helps us do it when
+    # it would be needed
+    class Proxy < SimpleDelegator
+      # Timeout on the watermark query
+      WATERMARK_REQUEST_TIMEOUT = 5_000
+
+      # Timeout on the TPL request query
+      TPL_REQUEST_TIMEOUT = 5_000
+
+      # How many attempts we want to take for something that would end up with all_brokers_down
+      BROKERS_DOWN_MAX_ATTEMPTS = 3
+
+      # How long should we wait in between all_brokers_down
+      BROKERS_DOWN_BACKOFF_TIME = 1
+
+      private_constant :WATERMARK_REQUEST_TIMEOUT, :BROKERS_DOWN_MAX_ATTEMPTS,
+                       :BROKERS_DOWN_BACKOFF_TIME, :TPL_REQUEST_TIMEOUT
+
+      attr_accessor :wrapped
+
+      alias __getobj__ wrapped
+
+      # @param obj [Rdkafka::Consumer, Proxy] rdkafka consumer or consumer wrapped with proxy
+      def initialize(obj)
+        super
+        # Do not allow for wrapping proxy with a proxy. This will prevent a case where we might
+        # wrap an already wrapped object with another proxy level. Simplifies passing consumers
+        # and makes it safe to wrap without type checking
+        @wrapped = obj.is_a?(self.class) ? obj.wrapped : obj
+      end
+
+      # Proxies the `#query_watermark_offsets` with extra recovery from timeout problems.
+      # We impose our own custom timeout to make sure, that high-latency clusters and overloaded
+      # clusters can handle our requests.
+      #
+      # @param topic [String] topic name
+      # @param partition [Partition]
+      # @return [Array<Integer, Integer>] watermark offsets
+      def query_watermark_offsets(topic, partition)
+        with_brokers_down_retry do
+          @wrapped.query_watermark_offsets(
+            topic,
+            partition,
+            WATERMARK_REQUEST_TIMEOUT
+          )
+        end
+      end
+
+      # Similar to `#query_watermark_offsets`, this method can be sensitive to latency. We handle
+      # this the same way
+      #
+      # @param tpl [Rdkafka::Consumer::TopicPartitionList] tpl to get time offsets
+      # @return [Rdkafka::Consumer::TopicPartitionList] tpl with time offsets
+      def offsets_for_times(tpl)
+        with_brokers_down_retry do
+          @wrapped.offsets_for_times(tpl, TPL_REQUEST_TIMEOUT)
+        end
+      end
+
+      private
+
+      # Runs expected block of code with few retries on all_brokers_down
+      # librdkafka can return `all_brokers_down` for scenarios when broker is overloaded or not
+      # reachable due to latency.
+      def with_brokers_down_retry
+        attempt ||= 0
+        attempt += 1
+
+        yield
+      rescue Rdkafka::RdkafkaError => e
+        raise if e.code != :all_brokers_down
+
+        if attempt <= BROKERS_DOWN_MAX_ATTEMPTS
+          sleep(BROKERS_DOWN_BACKOFF_TIME)
+
+          retry
+        end
+
+        raise
+      end
+    end
+  end
+end

data/lib/karafka/instrumentation/logger_listener.rb

@@ -280,6 +280,9 @@ module Karafka
         when 'connection.client.rebalance_callback.error'
           error "Rebalance callback error occurred: #{error}"
           error details
+        when 'connection.client.unsubscribe.error'
+          error "Client unsubscribe error occurred: #{error}"
+          error details
         else
           # This should never happen. Please contact the maintainers
           raise Errors::UnsupportedCaseError, event

data/lib/karafka/instrumentation/notifications.rb

@@ -35,6 +35,10 @@ module Karafka
         connection.listener.fetch_loop
         connection.listener.fetch_loop.received
 
+        connection.client.rebalance_callback
+        connection.client.poll.error
+        connection.client.unsubscribe.error
+
         consumer.consume
         consumer.consumed
         consumer.consuming.pause

data/lib/karafka/instrumentation/vendors/datadog/metrics_listener.rb

@@ -220,6 +220,11 @@ module Karafka
                   next if partition_name == '-1'
                   # Skip until lag info is available
                   next if partition_statistics['consumer_lag'] == -1
+                  next if partition_statistics['consumer_lag_stored'] == -1
+
+                  # Skip if we do not own the fetch assignment
+                  next if partition_statistics['fetch_state'] == 'stopped'
+                  next if partition_statistics['fetch_state'] == 'none'
 
                   public_send(
                     metric.type,

data/lib/karafka/patches/rdkafka/bindings.rb

@@ -1,7 +1,9 @@
 # frozen_string_literal: true
 
 module Karafka
+  # Namespace for third-party libraries patches
   module Patches
+    # Rdkafka patches specific to Karafka
     module Rdkafka
       # Binding patches that slightly change how rdkafka operates in certain places
       module Bindings
@@ -51,11 +53,18 @@ module Karafka
           # @param opaque [Rdkafka::Opaque]
           # @param tpl [Rdkafka::Consumer::TopicPartitionList]
           def trigger_callbacks(code, opaque, tpl)
-            case code
-            when RB::RD_KAFKA_RESP_ERR__ASSIGN_PARTITIONS
-              opaque.call_on_partitions_assigned(tpl)
-            when RB::RD_KAFKA_RESP_ERR__REVOKE_PARTITIONS
-              opaque.call_on_partitions_revoked(tpl)
+            Karafka.monitor.instrument(
+              'connection.client.rebalance_callback',
+              caller: self,
+              code: code,
+              tpl: tpl
+            ) do
+              case code
+              when RB::RD_KAFKA_RESP_ERR__ASSIGN_PARTITIONS
+                opaque.call_on_partitions_assigned(tpl)
+              when RB::RD_KAFKA_RESP_ERR__REVOKE_PARTITIONS
+                opaque.call_on_partitions_revoked(tpl)
+              end
             end
           rescue StandardError => e
             Karafka.monitor.instrument(

data/lib/karafka/pro/iterator/tpl_builder.rb

@@ -14,11 +14,6 @@
 module Karafka
   module Pro
     class Iterator
-      # Max time for a TPL request. We increase it to compensate for remote clusters latency
-      TPL_REQUEST_TIMEOUT = 2_000
-
-      private_constant :TPL_REQUEST_TIMEOUT
-
       # Because we have various formats in which we can provide the offsets, before we can
       # subscribe to them, there needs to be a bit of normalization.
       #
@@ -30,7 +25,7 @@ module Karafka
         # @param consumer [::Rdkafka::Consumer] consumer instance needed to talk with Kafka
         # @param expanded_topics [Hash] hash with expanded and normalized topics data
         def initialize(consumer, expanded_topics)
-          @consumer = consumer
+          @consumer = Connection::Proxy.new(consumer)
           @expanded_topics = expanded_topics
           @mapped_topics = Hash.new { |h, k| h[k] = {} }
         end
@@ -144,7 +139,7 @@ module Karafka
           # If there were no time-based, no need to query Kafka
           return if time_tpl.empty?
 
-          real_offsets = @consumer.offsets_for_times(time_tpl, TPL_REQUEST_TIMEOUT)
+          real_offsets = @consumer.offsets_for_times(time_tpl)
 
           real_offsets.to_h.each do |name, results|
             results.each do |result|

data/lib/karafka/pro/routing/features/pausing/contract.rb

@@ -0,0 +1,48 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Routing
+      module Features
+        class Pausing < Base
+          # Contract to make sure, that the pause settings on a per topic basis are as expected
+          class Contract < Contracts::Base
+            configure do |config|
+              config.error_messages = YAML.safe_load(
+                File.read(
+                  File.join(Karafka.gem_root, 'config', 'locales', 'pro_errors.yml')
+                )
+              ).fetch('en').fetch('validations').fetch('topic')
+
+              required(:pause_timeout) { |val| val.is_a?(Integer) && val.positive? }
+              required(:pause_max_timeout) { |val| val.is_a?(Integer) && val.positive? }
+              required(:pause_with_exponential_backoff) { |val| [true, false].include?(val) }
+
+              virtual do |data, errors|
+                next unless errors.empty?
+
+                pause_timeout = data.fetch(:pause_timeout)
+                pause_max_timeout = data.fetch(:pause_max_timeout)
+
+                next if pause_timeout <= pause_max_timeout
+
+                [[%i[pause_timeout], :max_timeout_vs_pause_max_timeout]]
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/routing/features/pausing/topic.rb

@@ -0,0 +1,44 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Routing
+      module Features
+        class Pausing < Base
+          # Expansion allowing for a per topic pause strategy definitions
+          module Topic
+            # Allows for per-topic pausing strategy setting
+            #
+            # @param timeout [Integer] how long should we wait upon processing error (milliseconds)
+            # @param max_timeout [Integer] what is the max timeout in case of an exponential
+            #   backoff (milliseconds)
+            # @param with_exponential_backoff [Boolean] should we use exponential backoff
+            #
+            # @note We do not construct here the nested config like we do with other routing
+            #   features, because this feature operates on the OSS layer by injection of values
+            #   and a nested config is not needed.
+            def pause(timeout: nil, max_timeout: nil, with_exponential_backoff: nil)
+              self.pause_timeout = timeout if timeout
+              self.pause_max_timeout = max_timeout if max_timeout
+
+              return unless with_exponential_backoff
+
+              self.pause_with_exponential_backoff = with_exponential_backoff
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/routing/features/pausing.rb

@@ -0,0 +1,25 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Routing
+      module Features
+        # Feature allowing for a per-route reconfiguration of the pausing strategy
+        # It can be useful when different topics should have different backoff policies
+        class Pausing < Base
+        end
+      end
+    end
+  end
+end

data/lib/karafka/processing/coordinators_buffer.rb

@@ -17,14 +17,18 @@ module Karafka
         @topics = topics
       end
 
-      # @param topic [String] topic name
+      # @param topic_name [String] topic name
       # @param partition [Integer] partition number
-      def find_or_create(topic, partition)
-        @coordinators[topic][partition] ||= @coordinator_class.new(
-          @topics.find(topic),
-          partition,
-          @pauses_manager.fetch(topic, partition)
-        )
+      def find_or_create(topic_name, partition)
+        @coordinators[topic_name][partition] ||= begin
+          routing_topic = @topics.find(topic_name)
+
+          @coordinator_class.new(
+            routing_topic,
+            partition,
+            @pauses_manager.fetch(routing_topic, partition)
+          )
+        end
       end
 
       # Resumes processing of partitions for which pause time has ended.
@@ -35,16 +39,16 @@ module Karafka
         @pauses_manager.resume(&block)
       end
 
-      # @param topic [String] topic name
+      # @param topic_name [String] topic name
       # @param partition [Integer] partition number
-      def revoke(topic, partition)
-        return unless @coordinators[topic].key?(partition)
+      def revoke(topic_name, partition)
+        return unless @coordinators[topic_name].key?(partition)
 
         # The fact that we delete here does not change the fact that the executor still holds the
         # reference to this coordinator. We delete it here, as we will no longer process any
         # new stuff with it and we may need a new coordinator if we regain this partition, but the
         # coordinator may still be in use
-        @coordinators[topic].delete(partition).revoke
+        @coordinators[topic_name].delete(partition).revoke
       end
 
       # Clears coordinators and re-created the pauses manager

data/lib/karafka/routing/topic.rb

@@ -18,6 +18,9 @@ module Karafka
         max_wait_time
         initial_offset
         consumer_persistence
+        pause_timeout
+        pause_max_timeout
+        pause_with_exponential_backoff
       ].freeze
 
       private_constant :INHERITABLE_ATTRIBUTES

data/lib/karafka/status.rb

@@ -62,5 +62,15 @@ module Karafka
         end
       end
     end
+
+    # @return [Boolean] true if we are in any of the status that would indicate we should no longer
+    #   process incoming data. It is a meta status built from others and not a separate state in
+    #   the sense of a state machine
+    def done?
+      # Short-track for the most common case not to invoke all others on normal execution
+      return false if running?
+
+      stopping? || stopped? || quieting? || quiet? || terminated?
+    end
   end
 end

data/lib/karafka/templates/karafka.rb.erb

@@ -43,7 +43,13 @@ class KarafkaApp < Karafka::App
   # This logger prints the producer development info using the Karafka logger.
   # It is similar to the consumer logger listener but producer oriented.
   Karafka.producer.monitor.subscribe(
-    WaterDrop::Instrumentation::LoggerListener.new(Karafka.logger)
+    WaterDrop::Instrumentation::LoggerListener.new(
+      # Log producer operations using the Karafka logger
+      Karafka.logger,
+      # If you set this to true, logs will contain each message details
+      # Please note, that this can be extensive
+      log_messages: false
+    )
   )
 
   routes.draw do

data/lib/karafka/version.rb

@@ -3,5 +3,5 @@
 # Main module namespace
 module Karafka
   # Current Karafka version
-  VERSION = '2.1.8'
+  VERSION = '2.1.10'
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: karafka
 version: !ruby/object:Gem::Version
-  version: 2.1.8
+  version: 2.1.10
 platform: ruby
 authors:
 - Maciej Mensfeld
@@ -12,30 +12,30 @@ cert_chain:
   -----BEGIN CERTIFICATE-----
   MIIEcDCCAtigAwIBAgIBATANBgkqhkiG9w0BAQsFADA/MRAwDgYDVQQDDAdjb250
   YWN0MRcwFQYKCZImiZPyLGQBGRYHa2FyYWZrYTESMBAGCgmSJomT8ixkARkWAmlv
-  MB4XDTIyMDgxOTE3MjEzN1oXDTIzMDgxOTE3MjEzN1owPzEQMA4GA1UEAwwHY29u
+  MB4XDTIzMDgyMTA3MjU1NFoXDTI0MDgyMDA3MjU1NFowPzEQMA4GA1UEAwwHY29u
   dGFjdDEXMBUGCgmSJomT8ixkARkWB2thcmFma2ExEjAQBgoJkiaJk/IsZAEZFgJp
-  bzCCAaIwDQYJKoZIhvcNAQEBBQADggGPADCCAYoCggGBAODzeO3L6lxdATzMHKNW
-  jFA/GGunoPuylO/BMzy8RiQHh7VIvysAKs0tHhTx3g2D0STDpF+hcQcPELFikiT2
-  F+1wOHj/SsrK7VKqfA8+gq04hKc5sQoX2Egf9k3V0YJ3eZ6R/koHkQ8A0TVt0w6F
-  ZQckoV4MqnEAx0g/FZN3mnHTlJ3VFLSBqJEIe+S6FZMl92mSv+hTrlUG8VaYxSfN
-  lTCvnKk284F6QZq5XIENLRmcDd/3aPBLnLwNnyMyhB+6gK8cUO+CFlDO5tjo/aBA
-  rUnl++wGG0JooF1ed0v+evOn9KoMBG6rHewcf79qJbVOscbD8qSAmo+sCXtcFryr
-  KRMTB8gNbowJkFRJDEe8tfRy11u1fYzFg/qNO82FJd62rKAw2wN0C29yCeQOPRb1
-  Cw9Y4ZwK9VFNEcV9L+3pHTHn2XfuZHtDaG198VweiF6raFO4yiEYccodH/USP0L5
-  cbcCFtmu/4HDSxL1ByQXO84A0ybJuk3/+aPUSXe9C9U8fwIDAQABo3cwdTAJBgNV
-  HRMEAjAAMAsGA1UdDwQEAwIEsDAdBgNVHQ4EFgQUSlcEakb7gfn/5E2WY6z73BF/
-  iZkwHQYDVR0RBBYwFIESY29udGFjdEBrYXJhZmthLmlvMB0GA1UdEgQWMBSBEmNv
-  bnRhY3RAa2FyYWZrYS5pbzANBgkqhkiG9w0BAQsFAAOCAYEA1aS+E7RXJ1w9g9mJ
-  G0NzFxe64OEuENosNlvYQCbRKGCXAU1qqelYkBQHseRgRKxLICrnypRo9IEobyHa
-  vDnJ4r7Tsb34dleqQW2zY/obG+cia3Ym2JsegXWF7dDOzCXJ4FN8MFoT2jHlqLLw
-  yrap0YO5zx0GSQ0Dwy8h2n2v2vanMEeCx7iNm3ERgR5WuN5sjzWoz2A/JLEEcK0C
-  EnAGKCWAd1fuG8IemDjT1edsd5FyYR4bIX0m+99oDuFZyPiiIbalmyYiSBBp59Yb
-  Q0P8zeBi4OfwCZNcxqz0KONmw9JLNv6DgyEAH5xe/4JzhMEgvIRiPj0pHfA7oqQF
-  KUNqvD1KlxbEC+bZfE5IZhnqYLdld/Ksqd22FI1RBhiS1Ejfsj99LVIm9cBuZEY2
-  Qf04B9ceLUaC4fPVEz10FyobjaFoY4i32xRto3XnrzeAgfEe4swLq8bQsR3w/EF3
-  MGU0FeSV2Yj7Xc2x/7BzLK8xQn5l7Yy75iPF+KP3vVmDHnNl
+  bzCCAaIwDQYJKoZIhvcNAQEBBQADggGPADCCAYoCggGBAOuZpyQKEwsTG9plLat7
+  8bUaNuNBEnouTsNMr6X+XTgvyrAxTuocdsyP1sNCjdS1B8RiiDH1/Nt9qpvlBWon
+  sdJ1SYhaWNVfqiYStTDnCx3PRMmHRdD4KqUWKpN6VpZ1O/Zu+9Mw0COmvXgZuuO9
+  wMSJkXRo6dTCfMedLAIxjMeBIxtoLR2e6Jm6MR8+8WYYVWrO9kSOOt5eKQLBY7aK
+  b/Dc40EcJKPg3Z30Pia1M9ZyRlb6SOj6SKpHRqc7vbVQxjEw6Jjal1lZ49m3YZMd
+  ArMAs9lQZNdSw5/UX6HWWURLowg6k10RnhTUtYyzO9BFev0JFJftHnmuk8vtb+SD
+  5VPmjFXg2VOcw0B7FtG75Vackk8QKfgVe3nSPhVpew2CSPlbJzH80wChbr19+e3+
+  YGr1tOiaJrL6c+PNmb0F31NXMKpj/r+n15HwlTMRxQrzFcgjBlxf2XFGnPQXHhBm
+  kp1OFnEq4GG9sON4glRldkwzi/f/fGcZmo5fm3d+0ZdNgwIDAQABo3cwdTAJBgNV
+  HRMEAjAAMAsGA1UdDwQEAwIEsDAdBgNVHQ4EFgQUPVH5+dLA80A1kJ2Uz5iGwfOa
+  1+swHQYDVR0RBBYwFIESY29udGFjdEBrYXJhZmthLmlvMB0GA1UdEgQWMBSBEmNv
+  bnRhY3RAa2FyYWZrYS5pbzANBgkqhkiG9w0BAQsFAAOCAYEAnpa0jcN7JzREHMTQ
+  bfZ+xcvlrzuROMY6A3zIZmQgbnoZZNuX4cMRrT1p1HuwXpxdpHPw7dDjYqWw3+1h
+  3mXLeMuk7amjQpYoSWU/OIZMhIsARra22UN8qkkUlUj3AwTaChVKN/bPJOM2DzfU
+  kz9vUgLeYYFfQbZqeI6SsM7ltilRV4W8D9yNUQQvOxCFxtLOetJ00fC/E7zMUzbK
+  IBwYFQYsbI6XQzgAIPW6nGSYKgRhkfpmquXSNKZRIQ4V6bFrufa+DzD0bt2ZA3ah
+  fMmJguyb5L2Gf1zpDXzFSPMG7YQFLzwYz1zZZvOU7/UCpQsHpID/YxqDp4+Dgb+Y
+  qma0whX8UG/gXFV2pYWpYOfpatvahwi+A1TwPQsuZwkkhi1OyF1At3RY+hjSXyav
+  AnG1dJU+yL2BK7vaVytLTstJME5mepSZ46qqIJXMuWob/YPDmVaBF39TDSG9e34s
+  msG3BiCqgOgHAnL23+CN3Rt8MsuRfEtoTKpJVcCfoEoNHOkc
   -----END CERTIFICATE-----
-date: 2023-07-29 00:00:00.000000000 Z
+date: 2023-08-21 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: karafka-core
@@ -77,7 +77,7 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 2.6.2
+        version: 2.6.6
     - - "<"
       - !ruby/object:Gem::Version
         version: 3.0.0
@@ -87,7 +87,7 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 2.6.2
+        version: 2.6.6
     - - "<"
       - !ruby/object:Gem::Version
         version: 3.0.0
@@ -178,6 +178,7 @@ files:
 - lib/karafka/connection/listeners_batch.rb
 - lib/karafka/connection/messages_buffer.rb
 - lib/karafka/connection/pauses_manager.rb
+- lib/karafka/connection/proxy.rb
 - lib/karafka/connection/raw_messages_buffer.rb
 - lib/karafka/connection/rebalance_manager.rb
 - lib/karafka/contracts.rb
@@ -314,6 +315,9 @@ files:
 - lib/karafka/pro/routing/features/long_running_job/config.rb
 - lib/karafka/pro/routing/features/long_running_job/contract.rb
 - lib/karafka/pro/routing/features/long_running_job/topic.rb
+- lib/karafka/pro/routing/features/pausing.rb
+- lib/karafka/pro/routing/features/pausing/contract.rb
+- lib/karafka/pro/routing/features/pausing/topic.rb
 - lib/karafka/pro/routing/features/throttling.rb
 - lib/karafka/pro/routing/features/throttling/config.rb
 - lib/karafka/pro/routing/features/throttling/contract.rb

metadata.gz.sig

@@ -1 +1,4 @@
-�߼$tJt�po�n��K����>���9mK�*��=��pc�3��;����;zP���{ȁ½������Iԋ���ɹX*�����:`��	�>]�I׉��GA���w
N�^E���oZO��M�͔�+�`jlX����Z^ֿ��S�S��\ꡛ��–��-�f�HU@����1� WSn�H.�KfN��[������v+��Xt����A:��lO0hy��x3�]?�|(���
k��C+�	Xf���B�١����L!��}�(@tl��G����>)ߙ������Rxܦ�)5E�>�&-�
Z���l����ç���<��@cػM;�?ۗ��郆p)6��N��\�6ÄD�C��G?��~c��%
+дy���]���G�mM�R#�u��Y�N���SF|�
F#�h@��>��=4���h��6�|�)�+Re�2�$�95*]�s<�2T;E ���|)K6��{G􈒟ʪU�L����:���:T�[�M���_�ZGi��y9;��B�b0u�P�Pi岥��A~ΰ�(��}��Iq@Q��kS�c
+7����
+�����].��`�+g��41*,$[��@���B8>���v ����dy9�'2
+�6Fn�v?����'�1ߛNzgzV��ɦ�հS�.�P���̑�UPH\�0���H7���:Ɨ(0���͜�|�`�R��[`xm�����Jt0
�n���ܝӶv�-���*s��A1�Iq:�"�AY�