karafka 2.4.14 → 2.4.15

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 22f45da117cf90a2ecbec04dbcaf39634b1e1b85e00f521d4880954c65268ecf
-  data.tar.gz: ce5318aaa8f52954a80981662a41ad17ab314e9706a00f2edf3e840604b87b32
+  metadata.gz: 5b49de31147b2a64d8927c91e25e6b607fdf1a8a7280b7cc3ba8c3663cf96b6f
+  data.tar.gz: 4a3d9e439e21b79b9ac6a2c8bf4e76219f970be1f6bb7316feb5c7f2f792271e
 SHA512:
-  metadata.gz: d8ec82f91aea2bdba595fa290feec1ca0b25dbd73cd0eafac6554538d3506c47ad8db42036ed838b199a3016341bf67d6daf4f03dfced3ddcf783850c0dc97bf
-  data.tar.gz: 2b750fce3294dda031f5a97bf5343711dcf52d1dbc00c61e029b602b302a549e9a70bb6ac4e952842e1ad840d7dd7b28115ed72118aad54917497672593de133
+  metadata.gz: 527a1c169ddc5a0f978f69dc3dbae6d30288997d2cfc95104ad6343d7a468c67afbde4c55051c1e7902d2c161fc1e615f8d703b9dfba123f734b9fbd7ccec5d1
+  data.tar.gz: a895657a8dece0c9bdb65afeeb00bc8df2e03e1957d345bc71ceab9d6cce74e5d0b4afcc251e26dfe230778a05b8742e40505a779695325844c6be630307ad82

data/CHANGELOG.md

@@ -1,5 +1,9 @@
 # Karafka Framework Changelog
 
+## 2.4.15 (2024-12-04)
+- [Fix] Assignment tracker current state fetch during a rebalance loop can cause an error on multi CG setup.
+- [Fix] Prevent double post-transaction offset dispatch to Kafka.
+
 ## 2.4.14 (2024-11-25)
 - [Enhancement] Improve low-level critical error reporting.
 - [Enhancement] Expand Kubernetes Liveness state reporting with critical errors detection.

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    karafka (2.4.14)
+    karafka (2.4.15)
       base64 (~> 0.2)
       karafka-core (>= 2.4.4, < 2.5.0)
       karafka-rdkafka (>= 0.17.2)
@@ -49,8 +49,8 @@ GEM
       activesupport (>= 6.1)
     i18n (1.14.6)
       concurrent-ruby (~> 1.0)
-    karafka-core (2.4.4)
-      karafka-rdkafka (>= 0.15.0, < 0.18.0)
+    karafka-core (2.4.5)
+      karafka-rdkafka (>= 0.17.6, < 0.18.0)
     karafka-rdkafka (0.17.6)
       ffi (~> 1.15)
       mini_portile2 (~> 2.6)
@@ -58,18 +58,18 @@ GEM
     karafka-testing (2.4.6)
       karafka (>= 2.4.0, < 2.5.0)
       waterdrop (>= 2.7.0)
-    karafka-web (0.10.3)
+    karafka-web (0.10.4)
       erubi (~> 1.4)
       karafka (>= 2.4.10, < 2.5.0)
       karafka-core (>= 2.4.0, < 2.5.0)
       roda (~> 3.68, >= 3.69)
       tilt (~> 2.0)
     logger (1.6.1)
-    mini_portile2 (2.8.7)
+    mini_portile2 (2.8.8)
     minitest (5.25.1)
     ostruct (0.6.1)
     raabro (1.4.0)
-    rack (3.1.7)
+    rack (3.1.8)
     rake (13.2.1)
     roda (3.84.0)
       rack

data/docker-compose.yml

@@ -3,7 +3,7 @@ version: '2'
 services:
   kafka:
     container_name: kafka
-    image: confluentinc/cp-kafka:7.7.1
+    image: confluentinc/cp-kafka:7.8.0
 
     ports:
       - 9092:9092

data/lib/karafka/instrumentation/assignments_tracker.rb

@@ -31,8 +31,13 @@ module Karafka
       def current
         assignments = {}
 
-        @assignments.each do |topic, partitions|
-          assignments[topic] = partitions.dup.freeze
+        # Since the `@assignments` state can change during a rebalance, if we would iterate over
+        # it exactly during state change, we would end up with the following error:
+        #   RuntimeError: can't add a new key into hash during iteration
+        @mutex.synchronize do
+          @assignments.each do |topic, partitions|
+            assignments[topic] = partitions.dup.freeze
+          end
         end
 
         assignments.freeze

data/lib/karafka/pro/processing/strategies/default.rb

@@ -63,6 +63,16 @@ module Karafka
               # Ignore earlier offsets than the one we already committed
               return true if coordinator.seek_offset > message.offset
               return false if revoked?
+
+              # If we have already marked this successfully in a transaction that was running
+              # we should not mark it again with the client offset delegation but instead we should
+              # just align the in-memory state
+              if @_in_transaction_marked
+                coordinator.seek_offset = message.offset + 1
+
+                return true
+              end
+
               return revoked? unless client.mark_as_consumed(message, offset_metadata)
 
               coordinator.seek_offset = message.offset + 1
@@ -90,6 +100,12 @@ module Karafka
               return true if coordinator.seek_offset > message.offset
               return false if revoked?
 
+              if @_in_transaction_marked
+                coordinator.seek_offset = message.offset + 1
+
+                return true
+              end
+
               return revoked? unless client.mark_as_consumed!(message, offset_metadata)
 
               coordinator.seek_offset = message.offset + 1
@@ -132,6 +148,7 @@ module Karafka
             transaction_started = true
             @_transaction_marked = []
             @_in_transaction = true
+            @_in_transaction_marked = false
 
             producer.transaction(&block)
 
@@ -143,6 +160,11 @@ module Karafka
             # @note We never need to use the blocking `#mark_as_consumed!` here because the offset
             #   anyhow was already stored during the transaction
             #
+            # @note Since the offset could have been already stored in Kafka (could have because
+            #   you can have transactions without marking), we use the `@_in_transaction_marked`
+            #   state to decide if we need to dispatch the offset via client at all
+            #   (if post transaction, then we do not have to)
+            #
             # @note In theory we could only keep reference to the most recent marking and reject
             #   others. We however do not do it for two reasons:
             #   - User may have non standard flow relying on some alternative order and we want to
@@ -158,6 +180,7 @@ module Karafka
             if transaction_started
               @_transaction_marked.clear
               @_in_transaction = false
+              @_in_transaction_marked = false
             end
           end
 
@@ -178,6 +201,7 @@ module Karafka
               offset_metadata
             )
 
+            @_in_transaction_marked = true
             @_transaction_marked ||= []
             @_transaction_marked << [message, offset_metadata, async]
           end

data/lib/karafka/routing/builder.rb

@@ -67,6 +67,20 @@ module Karafka
         end
       end
 
+      # Clear out the drawn routes.
+      alias array_clear clear
+      private :array_clear
+
+      # Clear routes and draw them again with the given block. Helpful for testing purposes.
+      # @param block [Proc] block we will evaluate within the builder context
+      def redraw(&block)
+        @mutex.synchronize do
+          @draws.clear
+          array_clear
+        end
+        draw(&block)
+      end
+
       # @return [Array<Karafka::Routing::ConsumerGroup>] only active consumer groups that
       #   we want to use. Since Karafka supports multi-process setup, we need to be able
       #   to pick only those consumer groups that should be active in our given process context
@@ -79,7 +93,7 @@ module Karafka
         @mutex.synchronize do
           @defaults = EMPTY_DEFAULTS
           @draws.clear
-          super
+          array_clear
         end
       end
 

data/lib/karafka/version.rb

@@ -3,5 +3,5 @@
 # Main module namespace
 module Karafka
   # Current Karafka version
-  VERSION = '2.4.14'
+  VERSION = '2.4.15'
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: karafka
 version: !ruby/object:Gem::Version
-  version: 2.4.14
+  version: 2.4.15
 platform: ruby
 authors:
 - Maciej Mensfeld
@@ -35,7 +35,7 @@ cert_chain:
   i9zWxov0mr44TWegTVeypcWGd/0nxu1+QHVNHJrpqlPBRvwQsUm7fwmRInGpcaB8
   ap8wNYvryYzrzvzUxIVFBVM5PacgkFqRmolCa8I7tdKQN+R1
   -----END CERTIFICATE-----
-date: 2024-11-25 00:00:00.000000000 Z
+date: 2024-12-04 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: base64