karafka 2.1.10 → 2.1.12

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 78db49b9cd20426b92ad1a3417b9bc2e79e9efad7f7d11e42873ffda75b850c2
-  data.tar.gz: 40b2a338c11ad9d6d07fdb0b82f422fe8f64417805db85a9bd111f9bce29039f
+  metadata.gz: de7a5880846f3b3cdab696683f4753bebdb6d133648297e930a415949937d1f5
+  data.tar.gz: df12badb044151bccbecb3832302182794af543b1740d9a939f984b299bd1521
 SHA512:
-  metadata.gz: a522abfbe63ebde9255032b5c6760016a014b8952f8c3f593c754479921408451ac0aeca632ba32d1fcefac78acd97a110e777949f6e70587d454febf5ad34f5
-  data.tar.gz: d12c97a4829bb0278041ce20e38f21c4894c77a7c9d4b3e4c1c222c502632fe03b0cce3d224f4b8c28ec3743c9c51e41c98a53310761687e92ce587445eae300
+  metadata.gz: 17fb8af9c36c3df7e6ef084aab4de9d796710cf6a324cbbac4be29a57dc09f254bac817ec0b44e2a0cf0d2b8aae69ab3834eb59c93bce4d4a540d8000808f31f
+  data.tar.gz: 8fedb0f5dfe436bd3854caf5a4ac0da367345be98e059e47ca24dd9d31d414aaf25debc66533965f9fdbdd3b5a78aeb4d719929b1ee274f5f3a36b3737ee2393

data/CHANGELOG.md

@@ -1,5 +1,12 @@
 # Karafka framework changelog
 
+## 2.1.12 (2023-08-25)
+- [Fix] Fix a case where DLQ + VP without intermediate marking would mark earlier message then the last one.
+
+## 2.1.11 (2023-08-23)
+- [Enhancement] Expand the error handling for offset related queries with timeout error retries.
+- [Enhancement] Allow for connection proxy timeouts configuration.
+
 ## 2.1.10 (2023-08-21)
 - [Enhancement] Introduce `connection.client.rebalance_callback` event for instrumentation of rebalances.
 - [Refactor] Introduce low level commands proxy to handle deviation in how we want to run certain commands and how rdkafka-ruby runs that by design.

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    karafka (2.1.10)
+    karafka (2.1.12)
       karafka-core (>= 2.1.1, < 2.2.0)
       thor (>= 0.20)
       waterdrop (>= 2.6.6, < 3.0.0)
@@ -33,7 +33,7 @@ GEM
     karafka-core (2.1.1)
       concurrent-ruby (>= 1.1)
       karafka-rdkafka (>= 0.13.1, < 0.14.0)
-    karafka-rdkafka (0.13.3)
+    karafka-rdkafka (0.13.4)
       ffi (~> 1.15)
       mini_portile2 (~> 2.6)
       rake (> 12)

data/config/locales/errors.yml

@@ -26,6 +26,12 @@ en:
       internal.process_format: needs to be present
       internal.routing.builder_format: needs to be present
       internal.routing.subscription_groups_builder_format: needs to be present
+      internal.connection.proxy.query_watermark_offsets.timeout_format: needs to be an integer bigger than 0
+      internal.connection.proxy.query_watermark_offsets.max_attempts_format: needs to be an integer bigger than 0
+      internal.connection.proxy.query_watermark_offsets.wait_time_format: needs to be an integer bigger than 0
+      internal.connection.proxy.offsets_for_times.timeout_format: needs to be an integer bigger than 0
+      internal.connection.proxy.offsets_for_times.max_attempts_format: needs to be an integer bigger than 0
+      internal.connection.proxy.offsets_for_times.wait_time_format: needs to be an integer bigger than 0
       key_must_be_a_symbol: All keys under the kafka settings scope need to be symbols
       max_timeout_vs_pause_max_timeout: pause_timeout must be less or equal to pause_max_timeout
       shutdown_timeout_vs_max_wait_time: shutdown_timeout must be more than max_wait_time

data/lib/karafka/connection/proxy.rb

@@ -10,20 +10,13 @@ module Karafka
     # do still want to be able to alter some functionalities. This wrapper helps us do it when
     # it would be needed
     class Proxy < SimpleDelegator
-      # Timeout on the watermark query
-      WATERMARK_REQUEST_TIMEOUT = 5_000
+      # Errors on which we want to retry
+      RETRYABLE_ERRORS = %i[
+        all_brokers_down
+        timed_out
+      ].freeze
 
-      # Timeout on the TPL request query
-      TPL_REQUEST_TIMEOUT = 5_000
-
-      # How many attempts we want to take for something that would end up with all_brokers_down
-      BROKERS_DOWN_MAX_ATTEMPTS = 3
-
-      # How long should we wait in between all_brokers_down
-      BROKERS_DOWN_BACKOFF_TIME = 1
-
-      private_constant :WATERMARK_REQUEST_TIMEOUT, :BROKERS_DOWN_MAX_ATTEMPTS,
-                       :BROKERS_DOWN_BACKOFF_TIME, :TPL_REQUEST_TIMEOUT
+      private_constant :RETRYABLE_ERRORS
 
       attr_accessor :wrapped
 
@@ -36,6 +29,7 @@ module Karafka
         # wrap an already wrapped object with another proxy level. Simplifies passing consumers
         # and makes it safe to wrap without type checking
         @wrapped = obj.is_a?(self.class) ? obj.wrapped : obj
+        @config = ::Karafka::App.config.internal.connection.proxy
       end
 
       # Proxies the `#query_watermark_offsets` with extra recovery from timeout problems.
@@ -46,12 +40,14 @@ module Karafka
       # @param partition [Partition]
       # @return [Array<Integer, Integer>] watermark offsets
       def query_watermark_offsets(topic, partition)
-        with_brokers_down_retry do
-          @wrapped.query_watermark_offsets(
-            topic,
-            partition,
-            WATERMARK_REQUEST_TIMEOUT
-          )
+        l_config = @config.query_watermark_offsets
+
+        with_broker_errors_retry(
+          # required to be in seconds, not ms
+          wait_time: l_config.wait_time / 1_000.to_f,
+          max_attempts: l_config.max_attempts
+        ) do
+          @wrapped.query_watermark_offsets(topic, partition, l_config.timeout)
         end
       end
 
@@ -61,8 +57,14 @@ module Karafka
       # @param tpl [Rdkafka::Consumer::TopicPartitionList] tpl to get time offsets
       # @return [Rdkafka::Consumer::TopicPartitionList] tpl with time offsets
       def offsets_for_times(tpl)
-        with_brokers_down_retry do
-          @wrapped.offsets_for_times(tpl, TPL_REQUEST_TIMEOUT)
+        l_config = @config.offsets_for_times
+
+        with_broker_errors_retry(
+          # required to be in seconds, not ms
+          wait_time: l_config.wait_time / 1_000.to_f,
+          max_attempts: l_config.max_attempts
+        ) do
+          @wrapped.offsets_for_times(tpl, l_config.timeout)
         end
       end
 
@@ -71,16 +73,20 @@ module Karafka
       # Runs expected block of code with few retries on all_brokers_down
       # librdkafka can return `all_brokers_down` for scenarios when broker is overloaded or not
       # reachable due to latency.
-      def with_brokers_down_retry
+      # @param max_attempts [Integer] how many attempts (not retries) should we take before failing
+      #   completely.
+      # @param wait_time [Integer, Float] how many seconds should we wait. It uses `#sleep` of Ruby
+      #   so it needs time in seconds.
+      def with_broker_errors_retry(max_attempts:, wait_time: 1)
         attempt ||= 0
         attempt += 1
 
         yield
       rescue Rdkafka::RdkafkaError => e
-        raise if e.code != :all_brokers_down
+        raise unless RETRYABLE_ERRORS.include?(e.code)
 
-        if attempt <= BROKERS_DOWN_MAX_ATTEMPTS
-          sleep(BROKERS_DOWN_BACKOFF_TIME)
+        if attempt <= max_attempts
+          sleep(wait_time)
 
           retry
         end

data/lib/karafka/contracts/config.rb

@@ -39,6 +39,22 @@ module Karafka
         required(:status) { |val| !val.nil? }
         required(:process) { |val| !val.nil? }
 
+        nested(:connection) do
+          nested(:proxy) do
+            nested(:query_watermark_offsets) do
+              required(:timeout) { |val| val.is_a?(Integer) && val.positive? }
+              required(:max_attempts) { |val| val.is_a?(Integer) && val.positive? }
+              required(:wait_time) { |val| val.is_a?(Integer) && val.positive? }
+            end
+
+            nested(:offsets_for_times) do
+              required(:timeout) { |val| val.is_a?(Integer) && val.positive? }
+              required(:max_attempts) { |val| val.is_a?(Integer) && val.positive? }
+              required(:wait_time) { |val| val.is_a?(Integer) && val.positive? }
+            end
+          end
+        end
+
         nested(:routing) do
           required(:builder) { |val| !val.nil? }
           required(:subscription_groups_builder) { |val| !val.nil? }

data/lib/karafka/pro/processing/strategies/dlq/default.rb

@@ -28,7 +28,7 @@ module Karafka
 
             # When we encounter non-recoverable message, we skip it and go on with our lives
             def handle_after_consume
-              coordinator.on_finished do
+              coordinator.on_finished do |last_group_message|
                 return if revoked?
 
                 if coordinator.success?
@@ -36,7 +36,7 @@ module Karafka
 
                   return if coordinator.manual_pause?
 
-                  mark_as_consumed(messages.last)
+                  mark_as_consumed(last_group_message)
                 elsif coordinator.pause_tracker.attempt <= topic.dead_letter_queue.max_retries
                   retry_after_pause
                 # If we've reached number of retries that we could, we need to skip the first

data/lib/karafka/pro/processing/strategies/dlq/ftr.rb

@@ -31,7 +31,7 @@ module Karafka
             # DLQ flow is standard here, what is not, is the success component where we need to
             # take into consideration the filtering
             def handle_after_consume
-              coordinator.on_finished do
+              coordinator.on_finished do |last_group_message|
                 return if revoked?
 
                 if coordinator.success?
@@ -39,7 +39,7 @@ module Karafka
 
                   return if coordinator.manual_pause?
 
-                  mark_as_consumed(messages.last)
+                  mark_as_consumed(last_group_message)
 
                   handle_post_filtering
                 elsif coordinator.pause_tracker.attempt <= topic.dead_letter_queue.max_retries

data/lib/karafka/setup/config.rb

@@ -126,6 +126,32 @@ module Karafka
           setting :activity_manager, default: Routing::ActivityManager.new
         end
 
+        # Namespace for internal connection related settings
+        setting :connection do
+          # Settings that are altered by our client proxy layer
+          setting :proxy do
+            # Watermark offsets request settings
+            setting :query_watermark_offsets do
+              # timeout for this request. For busy or remote clusters, this should be high enough
+              setting :timeout, default: 5_000
+              # How many times should we try to run this call before raising an error
+              setting :max_attempts, default: 3
+              # How long should we wait before next attempt in case of a failure
+              setting :wait_time, default: 1_000
+            end
+
+            # Offsets for times request settings
+            setting :offsets_for_times do
+              # timeout for this request. For busy or remote clusters, this should be high enough
+              setting :timeout, default: 5_000
+              # How many times should we try to run this call before raising an error
+              setting :max_attempts, default: 3
+              # How long should we wait before next attempt in case of a failure
+              setting :wait_time, default: 1_000
+            end
+          end
+        end
+
         setting :processing do
           # option scheduler [Object] scheduler we will be using
           setting :scheduler, default: Processing::Scheduler.new

data/lib/karafka/version.rb

@@ -3,5 +3,5 @@
 # Main module namespace
 module Karafka
   # Current Karafka version
-  VERSION = '2.1.10'
+  VERSION = '2.1.12'
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: karafka
 version: !ruby/object:Gem::Version
-  version: 2.1.10
+  version: 2.1.12
 platform: ruby
 authors:
 - Maciej Mensfeld
@@ -35,7 +35,7 @@ cert_chain:
   AnG1dJU+yL2BK7vaVytLTstJME5mepSZ46qqIJXMuWob/YPDmVaBF39TDSG9e34s
   msG3BiCqgOgHAnL23+CN3Rt8MsuRfEtoTKpJVcCfoEoNHOkc
   -----END CERTIFICATE-----
-date: 2023-08-21 00:00:00.000000000 Z
+date: 2023-08-25 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: karafka-core

metadata.gz.sig

@@ -1,4 +1,4 @@
-дy���]���G�mM�R#�u��Y�N���SF|�
F#�h@��>��=4���h��6�|�)�+Re�2�$�95*]�s<�2T;E ���|)K6��{G􈒟ʪU�L����:���:T�[�M���_�ZGi��y9;��B�b0u�P�Pi岥��A~ΰ�(��}��Iq@Q��kS�c
-7����
-�����].��`�+g��41*,$[��@���B8>���v ����dy9�'2
-�6Fn�v?����'�1ߛNzgzV��ɦ�հS�.�P���̑�UPH\�0���H7���:Ɨ(0���͜�|�`�R��[`xm�����Jt0
�n���ܝӶv�-���*s��A1�Iq:�"�AY�
+z�v�xq�]��98f�N��ա]���
+4W��x���
+�,����1��cG2b�Q�W�����0��πx��K�<ɛ�+x�5M��O�*����]��cZ�G���:��Z�ل�7!
�DB9>֐���
+�QX�P5�?|!H:�{#M�ߊKc0��^ؙ�����Q	�M�m�W���