karafka 2.4.4 → 2.4.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 1b7333a7812643bc128c721e411449dfd477adfa52968c7c291900e622cdcc04
-  data.tar.gz: 7062798152f0d4e097e5c4fcd228798103a680441579bbc1918c8bf5a65ce98b
+  metadata.gz: fe76e139b2007e11b0cb9393bc94e2b5b9cad5dc40d69b5ad2bfa34a0555e2b8
+  data.tar.gz: e288c63648d17388f577a45b51267e30fceb4726def0a42c40a9133de27d93f8
 SHA512:
-  metadata.gz: bd24bd450ac1cda02d18d7d487317139988b943ecc92c606cb713f13c819db044d6feae978873d26445e1ebf8d8823bedfd7efae58bc53c8213be9ef56a84c1d
-  data.tar.gz: 0f2f4b50ea90cd76f6275e0cdc9eeada5e5cd30bf6c18a219209dfe4b717d90ef88d7c6b68797e9eb0615c83ba03ee738d41164014ce74aaf27970ef07ba5974
+  metadata.gz: b5e46439a216e66c728ceb74070832407d39b1ebe06d21c858f3e3a0e8e599df15b749318131aa82a5bc994e6dd7ed3094f44639e706a57231f031f0e52186b6
+  data.tar.gz: dc44b6a2f0e5d21bdcbe156f0cc94c324d586c74e9fb757454157ac014897fc3cbd683acb19e6397d668f67a9c68e97d683521f1ceecdbf338e24f3d497c688d

data/.ruby-version

@@ -1 +1 @@
-3.3.3
+3.3.4

data/CHANGELOG.md

@@ -1,5 +1,13 @@
 # Karafka framework changelog
 
+## 2.4.5 (2024-07-18)
+- [Change] Inject `client.id` when building subscription group and not during the initial setup.
+- [Fix] Mitigate `confluentinc/librdkafka/issues/4783` by injecting dynamic client id when using `cooperative-sticky` strategy.
+
+### Change Note
+
+`client.id` is technically a low-importance value that should not (aside from this error) impact operations. This is why it is not considered a breaking change. This change may be reverted when the original issue is fixed in librdkafka.
+
 ## 2.4.4 (2024-07-04)
 - [Enhancement] Allow for offset storing from the Filtering API.
 - [Enhancement] Print more extensive error info on forceful shutdown.

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    karafka (2.4.4)
+    karafka (2.4.5)
       base64 (~> 0.2)
       karafka-core (>= 2.4.3, < 2.5.0)
       waterdrop (>= 2.7.3, < 3.0.0)

data/docker-compose.yml

@@ -3,7 +3,7 @@ version: '2'
 services:
   kafka:
     container_name: kafka
-    image: confluentinc/cp-kafka:7.6.1
+    image: confluentinc/cp-kafka:7.6.2
 
     ports:
       - 9092:9092

data/lib/karafka/routing/subscription_group.rb

@@ -121,8 +121,8 @@ module Karafka
         kafka = Setup::AttributesMap.consumer(@topics.first.kafka.dup)
 
         inject_group_instance_id(kafka)
+        inject_client_id(kafka)
 
-        kafka[:'client.id'] ||= client_id
         kafka[:'group.id'] ||= @consumer_group.id
         kafka[:'auto.offset.reset'] ||= @topics.first.initial_offset
         # Karafka manages the offsets based on the processing state, thus we do not rely on the
@@ -132,6 +132,26 @@ module Karafka
         kafka
       end
 
+      # Sets (if needed) the client.id attribute
+      #
+      # @param kafka [Hash] kafka level config
+      def inject_client_id(kafka)
+        # If client id is set directly on librdkafka level, we do nothing and just go with what
+        # end user has configured
+        return if kafka.key?(:'client.id')
+
+        # This mitigates an issue for multiplexing and potentially other cases when running
+        # multiple karafka processes on one machine, where librdkafka goes into an infinite
+        # loop when using cooperative-sticky and upscaling.
+        #
+        # @see https://github.com/confluentinc/librdkafka/issues/4783
+        kafka[:'client.id'] = if kafka[:'partition.assignment.strategy'] == 'cooperative-sticky'
+                                "#{client_id}/#{Time.now.to_f}/#{SecureRandom.hex[0..9]}"
+                              else
+                                client_id
+                              end
+      end
+
       # If we use static group memberships, there can be a case, where same instance id would
       # be set on many subscription groups as the group instance id from Karafka perspective is
       # set per config. Each instance even if they are subscribed to different topics needs to

data/lib/karafka/setup/config.rb

@@ -385,9 +385,6 @@ module Karafka
             config.kafka[key] = value
           end
 
-          # Use Karafka client_id as kafka client id if not set
-          config.kafka[:'client.id'] ||= config.client_id
-
           return if Karafka::App.env.production?
 
           KAFKA_DEV_DEFAULTS.each do |key, value|

data/lib/karafka/version.rb

@@ -3,5 +3,5 @@
 # Main module namespace
 module Karafka
   # Current Karafka version
-  VERSION = '2.4.4'
+  VERSION = '2.4.5'
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: karafka
 version: !ruby/object:Gem::Version
-  version: 2.4.4
+  version: 2.4.5
 platform: ruby
 authors:
 - Maciej Mensfeld
@@ -35,7 +35,7 @@ cert_chain:
   AnG1dJU+yL2BK7vaVytLTstJME5mepSZ46qqIJXMuWob/YPDmVaBF39TDSG9e34s
   msG3BiCqgOgHAnL23+CN3Rt8MsuRfEtoTKpJVcCfoEoNHOkc
   -----END CERTIFICATE-----
-date: 2024-07-04 00:00:00.000000000 Z
+date: 2024-07-18 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: base64