karafka 2.4.4 → 2.4.6

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 1b7333a7812643bc128c721e411449dfd477adfa52968c7c291900e622cdcc04
-  data.tar.gz: 7062798152f0d4e097e5c4fcd228798103a680441579bbc1918c8bf5a65ce98b
+  metadata.gz: e550fa0b7395eab3b961181319d195809baeca82c02935d2735cf98353f6ef8b
+  data.tar.gz: f734562bfb86db59a64f4fbeb596bfe126eacd428a4ade00da90d238d959ab64
 SHA512:
-  metadata.gz: bd24bd450ac1cda02d18d7d487317139988b943ecc92c606cb713f13c819db044d6feae978873d26445e1ebf8d8823bedfd7efae58bc53c8213be9ef56a84c1d
-  data.tar.gz: 0f2f4b50ea90cd76f6275e0cdc9eeada5e5cd30bf6c18a219209dfe4b717d90ef88d7c6b68797e9eb0615c83ba03ee738d41164014ce74aaf27970ef07ba5974
+  metadata.gz: f80b521d063653c93caee987358573c992af2488ffec9c6465dbb04f9e51e8334548cf669b2b5f8f5783540be01fa8b52e067042cf72afda35f14181148ec87d
+  data.tar.gz: 32fd0f08102a2687f4b646d15cd07599f9aaefac957f828396af1f2d91bc91d5c922119bfec04d94cba63af86724d43cb50852de140d78e289fdbb612892dc78

checksums.yaml.gz.sig

@@ -1 +1,4 @@
-zTS���?�D=���,��#,
+S ���#s�`&"�!p<�73�T�)��NU����3Ocv�1+.|�ò��9�n���g���S�u�{n )��G�"���>�<_�h�[o1�ECp<�|��\u&���O�{��BP��2�N>R*]��i�?k���.��Ua�V��P,P�X��,F�M�)+ᣦO1Lo�jI	���gr�
+�&"E�)*��+̰�FE�H��͗:�mk��1p� $R��Jz�������򵦙���(�D��Dt�8�W���
��
+�֝b���DV�(�C�*��s�����J��@k��fL���x���
+ה\��-���M����z���|EO��L�.-f���1�b��m~�X�-��&��X­8�

data/.ruby-version

@@ -1 +1 @@
-3.3.3
+3.3.4

data/CHANGELOG.md

@@ -1,4 +1,15 @@
-# Karafka framework changelog
+# Karafka Framework Changelog
+
+## 2.4.6 (2024-07-22)
+- [Fix] Mitigate `rd_kafka_cgrp_terminated` and other `librdkafka` shutdown issues by unsubscribing fully prior to shutdown.
+
+## 2.4.5 (2024-07-18)
+- [Change] Inject `client.id` when building subscription group and not during the initial setup.
+- [Fix] Mitigate `confluentinc/librdkafka/issues/4783` by injecting dynamic client id when using `cooperative-sticky` strategy.
+
+### Change Note
+
+`client.id` is technically a low-importance value that should not (aside from this error) impact operations. This is why it is not considered a breaking change. This change may be reverted when the original issue is fixed in librdkafka.
 
 ## 2.4.4 (2024-07-04)
 - [Enhancement] Allow for offset storing from the Filtering API.

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    karafka (2.4.4)
+    karafka (2.4.6)
       base64 (~> 0.2)
       karafka-core (>= 2.4.3, < 2.5.0)
       waterdrop (>= 2.7.3, < 3.0.0)
@@ -39,13 +39,13 @@ GEM
       activesupport (>= 6.1)
     i18n (1.14.5)
       concurrent-ruby (~> 1.0)
-    karafka-core (2.4.3)
-      karafka-rdkafka (>= 0.15.0, < 0.17.0)
-    karafka-rdkafka (0.16.0)
+    karafka-core (2.4.4)
+      karafka-rdkafka (>= 0.15.0, < 0.18.0)
+    karafka-rdkafka (0.17.0)
       ffi (~> 1.15)
       mini_portile2 (~> 2.6)
       rake (> 12)
-    karafka-testing (2.4.4)
+    karafka-testing (2.4.5)
       karafka (>= 2.4.0, < 2.5.0)
       waterdrop (>= 2.7.0)
     karafka-web (0.9.1)
@@ -58,7 +58,7 @@ GEM
     minitest (5.24.0)
     mutex_m (0.2.0)
     ostruct (0.6.0)
-    rack (3.1.4)
+    rack (3.1.5)
     rake (13.2.1)
     roda (3.81.0)
       rack
@@ -84,8 +84,8 @@ GEM
     tilt (2.4.0)
     tzinfo (2.0.6)
       concurrent-ruby (~> 1.0)
-    waterdrop (2.7.3)
-      karafka-core (>= 2.4.0, < 3.0.0)
+    waterdrop (2.7.4)
+      karafka-core (>= 2.4.3, < 3.0.0)
       karafka-rdkafka (>= 0.15.1)
       zeitwerk (~> 2.3)
     zeitwerk (2.6.16)
@@ -106,4 +106,4 @@ DEPENDENCIES
   simplecov
 
 BUNDLED WITH
-   2.5.14
+   2.5.11

data/docker-compose.yml

@@ -3,7 +3,7 @@ version: '2'
 services:
   kafka:
     container_name: kafka
-    image: confluentinc/cp-kafka:7.6.1
+    image: confluentinc/cp-kafka:7.6.2
 
     ports:
       - 9092:9092

data/lib/karafka/connection/client.rb

@@ -257,11 +257,15 @@ module Karafka
 
       # Gracefully stops topic consumption.
       def stop
-        # In case of cooperative-sticky, there is a bug in librdkafka that may hang it.
-        # To mitigate it we first need to unsubscribe so we will not receive any assignments and
-        # only then we should be good to go.
+        # librdkafka has several constant issues when shutting down during rebalance. This is
+        # an issue that gets back every few versions of librdkafka in a limited scope, for example
+        # for cooperative-sticky or in a general scope. This is why we unsubscribe and wait until
+        # we no longer have any assignments. That way librdkafka consumer shutdown should never
+        # happen with rebalance associated with the given consumer instance
+        #
+        # @see https://github.com/confluentinc/librdkafka/issues/4792
         # @see https://github.com/confluentinc/librdkafka/issues/4527
-        if @subscription_group.kafka[:'partition.assignment.strategy'] == 'cooperative-sticky'
+        if unsubscribe?
           unsubscribe
 
           until assignment.empty?
@@ -658,8 +662,13 @@ module Karafka
         subscriptions = @subscription_group.subscriptions
         assignments = @subscription_group.assignments(consumer)
 
-        consumer.subscribe(*subscriptions) if subscriptions
-        consumer.assign(assignments) if assignments
+        if subscriptions
+          consumer.subscribe(*subscriptions)
+          @mode = :subscribe
+        elsif assignments
+          consumer.assign(assignments)
+          @mode = :assign
+        end
 
         consumer
       end
@@ -690,6 +699,22 @@ module Karafka
         @kafka.start
         @kafka
       end
+
+      # Decides whether or not we should unsubscribe prior to closing.
+      #
+      # We cannot do it when there is a static group membership assignment as it would be
+      # reassigned.
+      # We cannot do it also for assign mode because then there are no subscriptions
+      # We also do not do it if there are no assignments at all as it does not make sense
+      #
+      # @return [Boolean] should we unsubscribe prior to shutdown
+      def unsubscribe?
+        return false if @subscription_group.kafka.key?(:'group.instance.id')
+        return false if @mode != :subscribe
+        return false if assignment.empty?
+
+        true
+      end
     end
   end
 end

data/lib/karafka/routing/subscription_group.rb

@@ -121,8 +121,8 @@ module Karafka
         kafka = Setup::AttributesMap.consumer(@topics.first.kafka.dup)
 
         inject_group_instance_id(kafka)
+        inject_client_id(kafka)
 
-        kafka[:'client.id'] ||= client_id
         kafka[:'group.id'] ||= @consumer_group.id
         kafka[:'auto.offset.reset'] ||= @topics.first.initial_offset
         # Karafka manages the offsets based on the processing state, thus we do not rely on the
@@ -132,6 +132,26 @@ module Karafka
         kafka
       end
 
+      # Sets (if needed) the client.id attribute
+      #
+      # @param kafka [Hash] kafka level config
+      def inject_client_id(kafka)
+        # If client id is set directly on librdkafka level, we do nothing and just go with what
+        # end user has configured
+        return if kafka.key?(:'client.id')
+
+        # This mitigates an issue for multiplexing and potentially other cases when running
+        # multiple karafka processes on one machine, where librdkafka goes into an infinite
+        # loop when using cooperative-sticky and upscaling.
+        #
+        # @see https://github.com/confluentinc/librdkafka/issues/4783
+        kafka[:'client.id'] = if kafka[:'partition.assignment.strategy'] == 'cooperative-sticky'
+                                "#{client_id}/#{Time.now.to_f}/#{SecureRandom.hex[0..9]}"
+                              else
+                                client_id
+                              end
+      end
+
       # If we use static group memberships, there can be a case, where same instance id would
       # be set on many subscription groups as the group instance id from Karafka perspective is
       # set per config. Each instance even if they are subscribed to different topics needs to

data/lib/karafka/setup/attributes_map.rb

@@ -38,6 +38,7 @@ module Karafka
         default_topic_conf
         enable.auto.commit
         enable.auto.offset.store
+        enable.metrics.push
         enable.partition.eof
         enable.random.seed
         enable.sasl.oauthbearer.unsecure.jwt
@@ -89,6 +90,8 @@ module Karafka
         reconnect.backoff.max.ms
         reconnect.backoff.ms
         resolve_cb
+        retry.backoff.max.ms
+        retry.backoff.ms
         sasl.kerberos.keytab
         sasl.kerberos.kinit.cmd
         sasl.kerberos.min.time.before.relogin
@@ -182,6 +185,7 @@ module Karafka
         dr_msg_cb
         enable.gapless.guarantee
         enable.idempotence
+        enable.metrics.push
         enable.random.seed
         enable.sasl.oauthbearer.unsecure.jwt
         enable.ssl.certificate.verification

data/lib/karafka/setup/config.rb

@@ -385,9 +385,6 @@ module Karafka
             config.kafka[key] = value
           end
 
-          # Use Karafka client_id as kafka client id if not set
-          config.kafka[:'client.id'] ||= config.client_id
-
           return if Karafka::App.env.production?
 
           KAFKA_DEV_DEFAULTS.each do |key, value|

data/lib/karafka/version.rb

@@ -3,5 +3,5 @@
 # Main module namespace
 module Karafka
   # Current Karafka version
-  VERSION = '2.4.4'
+  VERSION = '2.4.6'
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: karafka
 version: !ruby/object:Gem::Version
-  version: 2.4.4
+  version: 2.4.6
 platform: ruby
 authors:
 - Maciej Mensfeld
@@ -35,7 +35,7 @@ cert_chain:
   AnG1dJU+yL2BK7vaVytLTstJME5mepSZ46qqIJXMuWob/YPDmVaBF39TDSG9e34s
   msG3BiCqgOgHAnL23+CN3Rt8MsuRfEtoTKpJVcCfoEoNHOkc
   -----END CERTIFICATE-----
-date: 2024-07-04 00:00:00.000000000 Z
+date: 2024-07-22 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: base64