karafka 2.2.5 → 2.2.6

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 729320b972ba55fe500db0729aa651c6308e1975db33d3ebd4db8061561e38bc
-  data.tar.gz: f26fa836af6b7777692c7fd36fad1c89ce22c7ab60ab243662df40993acd7bd4
+  metadata.gz: 36fe50a9b0a758b9b916ca668c862085629bf09bcf16f7c0e8a340a9b7333abb
+  data.tar.gz: 61d3f10df4e1545a1a4e0d8190eb6ee2a7b751636bb347cf2703a74149fc710c
 SHA512:
-  metadata.gz: 86cecfe51fbdb21943f75a98a1ee8e755496f9777bf5c4445ad44ecff45fa5be34fc79c0ad6358ded3886a59ebde4f0cd47598fdcdda41f75c5152489cf5ca35
-  data.tar.gz: fec15466a63fc6e49089f339d3f0d9bb3d4d308c7855adf17f23e93ef10e3e09daf6e8453296a53bf74a57d1004883fc2b144c4564342daec8b94b9a7288bd82
+  metadata.gz: f12fb91f2e8e4e7dba766fbf32c79b734f5cd9f6fa63a66f80863a819dfd6d442024a97a4095bea84ef87bb0cb645ff431ba6d2f04f46d72864c377d61fc76a6
+  data.tar.gz: 4235c1944c98f0bd6359b82b7eab68a69812d70665ab0436c9f67cc5011e4c8edfa2c194a6d65744a4d82f97710beca5ddb871b8671febf61e5c9f591a746bee

data/CHANGELOG.md

@@ -1,5 +1,8 @@
 # Karafka framework changelog
 
+## 2.2.6 (2023-09-26)
+- [Enhancement] Retry `Karafka::Admin#read_watermark_offsets` fetching upon `not_leader_for_partition` that can occur mostly on newly created topics in KRaft and after crashes during leader selection.
+
 ## 2.2.5 (2023-09-25)
 - [Enhancement] Ensure, that when topic related operations end, the result is usable. There were few cases where admin operations on topics would finish successfully but internal Kafka caches would not report changes for a short period of time.
 - [Enhancement] Stabilize cooperative-sticky early shutdown procedure.

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    karafka (2.2.5)
+    karafka (2.2.6)
       karafka-core (>= 2.2.2, < 2.3.0)
       thor (>= 0.20)
       waterdrop (>= 2.6.6, < 3.0.0)

data/lib/karafka/admin.rb

@@ -142,7 +142,12 @@ module Karafka
       # @return [Array<Integer, Integer>] low watermark offset and high watermark offset
       def read_watermark_offsets(name, partition)
         with_consumer do |consumer|
-          consumer.query_watermark_offsets(name, partition)
+          # For newly created topics or in cases where we're trying to get them but there is no
+          # leader, this can fail. It happens more often for new topics under KRaft, however we
+          # still want to make sure things operate as expected even then
+          with_rdkafka_retry(codes: %i[not_leader_for_partition]) do
+            consumer.query_watermark_offsets(name, partition)
+          end
         end
       end
 
@@ -228,6 +233,31 @@ module Karafka
         raise
       end
 
+      # Handles retries for rdkafka related errors that we specify in `:codes`.
+      #
+      # Some operations temporarily fail, especially for cases where we changed something fast
+      # like topic creation or repartitioning. In cases like this it is ok to retry operations that
+      # do not change the state as it will usually recover.
+      #
+      # @param codes [Array<Symbol>] librdkafka error codes on which we want to retry
+      # @param max_attempts [Integer] number of attempts (including initial) after which we should
+      #   give up
+      #
+      # @note This code implements a simple backoff that increases with each attempt.
+      def with_rdkafka_retry(codes:, max_attempts: 5)
+        attempt ||= 0
+        attempt += 1
+
+        yield
+      rescue Rdkafka::RdkafkaError => e
+        raise unless codes.include?(e.code)
+        raise if attempt >= max_attempts
+
+        sleep(max_attempts)
+
+        retry
+      end
+
       # @param type [Symbol] type of config we want
       # @param settings [Hash] extra settings for config (if needed)
       # @return [::Rdkafka::Config] rdkafka config

data/lib/karafka/version.rb

@@ -3,5 +3,5 @@
 # Main module namespace
 module Karafka
   # Current Karafka version
-  VERSION = '2.2.5'
+  VERSION = '2.2.6'
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: karafka
 version: !ruby/object:Gem::Version
-  version: 2.2.5
+  version: 2.2.6
 platform: ruby
 authors:
 - Maciej Mensfeld
@@ -35,7 +35,7 @@ cert_chain:
   AnG1dJU+yL2BK7vaVytLTstJME5mepSZ46qqIJXMuWob/YPDmVaBF39TDSG9e34s
   msG3BiCqgOgHAnL23+CN3Rt8MsuRfEtoTKpJVcCfoEoNHOkc
   -----END CERTIFICATE-----
-date: 2023-09-25 00:00:00.000000000 Z
+date: 2023-09-26 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: karafka-core