karafka 2.4.5 → 2.4.6

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: fe76e139b2007e11b0cb9393bc94e2b5b9cad5dc40d69b5ad2bfa34a0555e2b8
-  data.tar.gz: e288c63648d17388f577a45b51267e30fceb4726def0a42c40a9133de27d93f8
+  metadata.gz: e550fa0b7395eab3b961181319d195809baeca82c02935d2735cf98353f6ef8b
+  data.tar.gz: f734562bfb86db59a64f4fbeb596bfe126eacd428a4ade00da90d238d959ab64
 SHA512:
-  metadata.gz: b5e46439a216e66c728ceb74070832407d39b1ebe06d21c858f3e3a0e8e599df15b749318131aa82a5bc994e6dd7ed3094f44639e706a57231f031f0e52186b6
-  data.tar.gz: dc44b6a2f0e5d21bdcbe156f0cc94c324d586c74e9fb757454157ac014897fc3cbd683acb19e6397d668f67a9c68e97d683521f1ceecdbf338e24f3d497c688d
+  metadata.gz: f80b521d063653c93caee987358573c992af2488ffec9c6465dbb04f9e51e8334548cf669b2b5f8f5783540be01fa8b52e067042cf72afda35f14181148ec87d
+  data.tar.gz: 32fd0f08102a2687f4b646d15cd07599f9aaefac957f828396af1f2d91bc91d5c922119bfec04d94cba63af86724d43cb50852de140d78e289fdbb612892dc78

data/CHANGELOG.md

@@ -1,4 +1,7 @@
-# Karafka framework changelog
+# Karafka Framework Changelog
+
+## 2.4.6 (2024-07-22)
+- [Fix] Mitigate `rd_kafka_cgrp_terminated` and other `librdkafka` shutdown issues by unsubscribing fully prior to shutdown.
 
 ## 2.4.5 (2024-07-18)
 - [Change] Inject `client.id` when building subscription group and not during the initial setup.

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    karafka (2.4.5)
+    karafka (2.4.6)
       base64 (~> 0.2)
       karafka-core (>= 2.4.3, < 2.5.0)
       waterdrop (>= 2.7.3, < 3.0.0)
@@ -39,13 +39,13 @@ GEM
       activesupport (>= 6.1)
     i18n (1.14.5)
       concurrent-ruby (~> 1.0)
-    karafka-core (2.4.3)
-      karafka-rdkafka (>= 0.15.0, < 0.17.0)
-    karafka-rdkafka (0.16.0)
+    karafka-core (2.4.4)
+      karafka-rdkafka (>= 0.15.0, < 0.18.0)
+    karafka-rdkafka (0.17.0)
       ffi (~> 1.15)
       mini_portile2 (~> 2.6)
       rake (> 12)
-    karafka-testing (2.4.4)
+    karafka-testing (2.4.5)
       karafka (>= 2.4.0, < 2.5.0)
       waterdrop (>= 2.7.0)
     karafka-web (0.9.1)
@@ -58,7 +58,7 @@ GEM
     minitest (5.24.0)
     mutex_m (0.2.0)
     ostruct (0.6.0)
-    rack (3.1.4)
+    rack (3.1.5)
     rake (13.2.1)
     roda (3.81.0)
       rack
@@ -84,8 +84,8 @@ GEM
     tilt (2.4.0)
     tzinfo (2.0.6)
       concurrent-ruby (~> 1.0)
-    waterdrop (2.7.3)
-      karafka-core (>= 2.4.0, < 3.0.0)
+    waterdrop (2.7.4)
+      karafka-core (>= 2.4.3, < 3.0.0)
       karafka-rdkafka (>= 0.15.1)
       zeitwerk (~> 2.3)
     zeitwerk (2.6.16)
@@ -106,4 +106,4 @@ DEPENDENCIES
   simplecov
 
 BUNDLED WITH
-   2.5.14
+   2.5.11

data/lib/karafka/connection/client.rb

@@ -257,11 +257,15 @@ module Karafka
 
       # Gracefully stops topic consumption.
       def stop
-        # In case of cooperative-sticky, there is a bug in librdkafka that may hang it.
-        # To mitigate it we first need to unsubscribe so we will not receive any assignments and
-        # only then we should be good to go.
+        # librdkafka has several constant issues when shutting down during rebalance. This is
+        # an issue that gets back every few versions of librdkafka in a limited scope, for example
+        # for cooperative-sticky or in a general scope. This is why we unsubscribe and wait until
+        # we no longer have any assignments. That way librdkafka consumer shutdown should never
+        # happen with rebalance associated with the given consumer instance
+        #
+        # @see https://github.com/confluentinc/librdkafka/issues/4792
         # @see https://github.com/confluentinc/librdkafka/issues/4527
-        if @subscription_group.kafka[:'partition.assignment.strategy'] == 'cooperative-sticky'
+        if unsubscribe?
           unsubscribe
 
           until assignment.empty?
@@ -658,8 +662,13 @@ module Karafka
         subscriptions = @subscription_group.subscriptions
         assignments = @subscription_group.assignments(consumer)
 
-        consumer.subscribe(*subscriptions) if subscriptions
-        consumer.assign(assignments) if assignments
+        if subscriptions
+          consumer.subscribe(*subscriptions)
+          @mode = :subscribe
+        elsif assignments
+          consumer.assign(assignments)
+          @mode = :assign
+        end
 
         consumer
       end
@@ -690,6 +699,22 @@ module Karafka
         @kafka.start
         @kafka
       end
+
+      # Decides whether or not we should unsubscribe prior to closing.
+      #
+      # We cannot do it when there is a static group membership assignment as it would be
+      # reassigned.
+      # We cannot do it also for assign mode because then there are no subscriptions
+      # We also do not do it if there are no assignments at all as it does not make sense
+      #
+      # @return [Boolean] should we unsubscribe prior to shutdown
+      def unsubscribe?
+        return false if @subscription_group.kafka.key?(:'group.instance.id')
+        return false if @mode != :subscribe
+        return false if assignment.empty?
+
+        true
+      end
     end
   end
 end

data/lib/karafka/setup/attributes_map.rb

@@ -38,6 +38,7 @@ module Karafka
         default_topic_conf
         enable.auto.commit
         enable.auto.offset.store
+        enable.metrics.push
         enable.partition.eof
         enable.random.seed
         enable.sasl.oauthbearer.unsecure.jwt
@@ -89,6 +90,8 @@ module Karafka
         reconnect.backoff.max.ms
         reconnect.backoff.ms
         resolve_cb
+        retry.backoff.max.ms
+        retry.backoff.ms
         sasl.kerberos.keytab
         sasl.kerberos.kinit.cmd
         sasl.kerberos.min.time.before.relogin
@@ -182,6 +185,7 @@ module Karafka
         dr_msg_cb
         enable.gapless.guarantee
         enable.idempotence
+        enable.metrics.push
         enable.random.seed
         enable.sasl.oauthbearer.unsecure.jwt
         enable.ssl.certificate.verification

data/lib/karafka/version.rb

@@ -3,5 +3,5 @@
 # Main module namespace
 module Karafka
   # Current Karafka version
-  VERSION = '2.4.5'
+  VERSION = '2.4.6'
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: karafka
 version: !ruby/object:Gem::Version
-  version: 2.4.5
+  version: 2.4.6
 platform: ruby
 authors:
 - Maciej Mensfeld
@@ -35,7 +35,7 @@ cert_chain:
   AnG1dJU+yL2BK7vaVytLTstJME5mepSZ46qqIJXMuWob/YPDmVaBF39TDSG9e34s
   msG3BiCqgOgHAnL23+CN3Rt8MsuRfEtoTKpJVcCfoEoNHOkc
   -----END CERTIFICATE-----
-date: 2024-07-18 00:00:00.000000000 Z
+date: 2024-07-22 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: base64