karafka 2.1.7 → 2.1.8

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: f405521c7a6706cc95e764a4740e7570935f7595d34481bbe33fb617e5537978
-  data.tar.gz: cd6671c441c07e31050bbddab290ba4d31e4a580a646cfd965edf58c19ff150c
+  metadata.gz: 042f365fb134a24ae360d678590ce798014751c8a23fb267001c920a42aa5324
+  data.tar.gz: ba2950de557a5f6c775577ce392d60ee839184f50b7d9225969c684625c9ecd0
 SHA512:
-  metadata.gz: 7b5e343a0d2c6e1f885c6eac6509de2f411b54e1a30ce12fac6fa18bb813d82ef666444345b92d8348ac4955cdabfc47ad3658312482f6c500ca169814f10517
-  data.tar.gz: 1b0c319f85dde3bc20b21a842da220d513351b436b3e4de08d56e69a02c36c7c2cd4187c879596ffc73f5dffc2cc3f032c6a8cdbd958ce34138866d27aa00b2b
+  metadata.gz: 30b5fcd92c348c50482cb84542380ad28b317d47e01efad2d2049cd3ba7872c5f66a7a4fde93d8bfa262d7fcb3745dedbdb89a4fd5e6cdf2288a43606ea2361d
+  data.tar.gz: d9c8ba95b2b71f46a3d35e2f5be634473a7aa9f45d9b5924e1019dcb53c7cea6aca8c594f9f4d3bf85a14176c3cb98a7167b4eb6e1f6f2192059d8040440e4a9

data/CHANGELOG.md

@@ -1,5 +1,12 @@
 # Karafka framework changelog
 
+## 2.1.8 (2023-07-29)
+- [Improvement] Introduce `Karafka::BaseConsumer#used?` method to indicate, that at least one invocation of `#consume` took or will take place. This can be used as a replacement to the non-direct `messages.count` check for shutdown and revocation to ensure, that the consumption took place or is taking place (in case of running LRJ).
+- [Improvement] Make `messages#to_a` return copy of the underlying array to prevent scenarios, where the mutation impacts offset management.
+- [Improvement] Mitigate a librdkafka `cooperative-sticky` rebalance crash issue.
+- [Improvement] Provide ability to overwrite `consumer_persistence` per subscribed topic. This is mostly useful for plugins and extensions developers.
+- [Fix] Fix a case where the performance tracker would crash in case of mutation of messages to an empty state.
+
 ## 2.1.7 (2023-07-22)
 - [Improvement] Always query for watermarks in the Iterator to improve the initial response time.
 - [Improvement] Add `max_wait_time` option to the Iterator.

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    karafka (2.1.7)
+    karafka (2.1.8)
       karafka-core (>= 2.1.1, < 2.2.0)
       thor (>= 0.20)
       waterdrop (>= 2.6.2, < 3.0.0)
@@ -37,13 +37,13 @@ GEM
       ffi (~> 1.15)
       mini_portile2 (~> 2.6)
       rake (> 12)
-    karafka-web (0.6.1)
+    karafka-web (0.6.3)
       erubi (~> 1.4)
       karafka (>= 2.1.4, < 3.0.0)
       karafka-core (>= 2.0.13, < 3.0.0)
       roda (~> 3.68, >= 3.68)
       tilt (~> 2.0)
-    mini_portile2 (2.8.2)
+    mini_portile2 (2.8.4)
     minitest (5.18.1)
     rack (3.0.8)
     rake (13.0.6)
@@ -72,7 +72,7 @@ GEM
     tilt (2.2.0)
     tzinfo (2.0.6)
       concurrent-ruby (~> 1.0)
-    waterdrop (2.6.4)
+    waterdrop (2.6.5)
       karafka-core (>= 2.1.1, < 3.0.0)
       zeitwerk (~> 2.3)
     zeitwerk (2.6.8)

data/lib/karafka/base_consumer.rb

@@ -25,6 +25,7 @@ module Karafka
     # Creates new consumer and assigns it an id
     def initialize
       @id = SecureRandom.hex(6)
+      @used = false
     end
 
     # Can be used to run preparation code prior to the job being enqueued
@@ -34,6 +35,7 @@ module Karafka
     #   not as a part of the public api. This should not perform any extensive operations as it is
     #   blocking and running in the listener thread.
     def on_before_enqueue
+      @used = true
       handle_before_enqueue
     rescue StandardError => e
       Karafka.monitor.instrument(
@@ -160,6 +162,14 @@ module Karafka
     # some teardown procedures (closing file handler, etc).
     def shutdown; end
 
+    # @return [Boolean] was this consumer in active use. Active use means running `#consume` at
+    #   least once. Consumer may have to run `#revoked` or `#shutdown` despite not running
+    #   `#consume` previously in delayed job cases and other cases that potentially involve running
+    #   the `Jobs::Idle` for house-keeping
+    def used?
+      @used
+    end
+
     # Pauses processing on a given offset for the current topic partition
     #
     # After given partition is resumed, it will continue processing from the given offset

data/lib/karafka/connection/client.rb

@@ -23,11 +23,17 @@ module Karafka
       # Max time for a TPL request. We increase it to compensate for remote clusters latency
       TPL_REQUEST_TIMEOUT = 2_000
 
+      # 1 minute of max wait for the first rebalance before a forceful attempt
+      # This applies only to a case when a short-lived Karafka instance with a client would be
+      # closed before first rebalance. Mitigates a librdkafka bug.
+      COOPERATIVE_STICKY_MAX_WAIT = 60_000
+
       # We want to make sure we never close several clients in the same moment to prevent
       # potential race conditions and other issues
       SHUTDOWN_MUTEX = Mutex.new
 
-      private_constant :MAX_POLL_RETRIES, :SHUTDOWN_MUTEX, :TPL_REQUEST_TIMEOUT
+      private_constant :MAX_POLL_RETRIES, :SHUTDOWN_MUTEX, :TPL_REQUEST_TIMEOUT,
+                       :COOPERATIVE_STICKY_MAX_WAIT
 
       # Creates a new consumer instance.
       #
@@ -226,6 +232,22 @@ module Karafka
       #   as until all the consumers are stopped, the server will keep running serving only
       #   part of the messages
       def stop
+        # This ensures, that we do not stop the underlying client until it passes the first
+        # rebalance for cooperative-sticky. Otherwise librdkafka may crash
+        #
+        # We set a timeout just in case the rebalance would never happen or would last for an
+        # extensive time period.
+        #
+        # @see https://github.com/confluentinc/librdkafka/issues/4312
+        if @subscription_group.kafka[:'partition.assignment.strategy'] == 'cooperative-sticky'
+          (COOPERATIVE_STICKY_MAX_WAIT / 100).times do
+            # If we're past the first rebalance, no need to wait
+            break if @rebalance_manager.active?
+
+            sleep(0.1)
+          end
+        end
+
         close
       end
 

data/lib/karafka/connection/rebalance_manager.rb

@@ -30,6 +30,7 @@ module Karafka
         @assigned_partitions = {}
         @revoked_partitions = {}
         @changed = false
+        @active = false
       end
 
       # Resets the rebalance manager state
@@ -46,11 +47,20 @@ module Karafka
         @changed
       end
 
+      # @return [Boolean] true if there was at least one rebalance
+      # @note This method is needed to make sure that when using cooperative-sticky, we do not
+      #   close until first rebalance. Otherwise librdkafka may crash.
+      # @see https://github.com/confluentinc/librdkafka/issues/4312
+      def active?
+        @active
+      end
+
       # Callback that kicks in inside of rdkafka, when new partitions are assigned.
       #
       # @private
       # @param partitions [Rdkafka::Consumer::TopicPartitionList]
       def on_partitions_assigned(partitions)
+        @active = true
         @assigned_partitions = partitions.to_h.transform_values { |part| part.map(&:partition) }
         @changed = true
       end
@@ -60,6 +70,7 @@ module Karafka
       # @private
       # @param partitions [Rdkafka::Consumer::TopicPartitionList]
       def on_partitions_revoked(partitions)
+        @active = true
         @revoked_partitions = partitions.to_h.transform_values { |part| part.map(&:partition) }
         @changed = true
       end

data/lib/karafka/messages/messages.rb

@@ -60,10 +60,12 @@ module Karafka
         @messages_array.size
       end
 
-      # @return [Array<Karafka::Messages::Message>] pure array with messages
+      # @return [Array<Karafka::Messages::Message>] copy of the pure array with messages
       def to_a
-        @messages_array
+        @messages_array.dup
       end
+
+      alias count size
     end
   end
 end

data/lib/karafka/processing/executor.rb

@@ -48,7 +48,7 @@ module Karafka
         # We reload the consumers with each batch instead of relying on some external signals
         # when needed for consistency. That way devs may have it on or off and not in this
         # middle state, where re-creation of a consumer instance would occur only sometimes
-        @consumer = nil unless ::Karafka::App.config.consumer_persistence
+        @consumer = nil unless topic.consumer_persistence
 
         # First we build messages batch...
         consumer.messages = Messages::Builders::Messages.call(

data/lib/karafka/routing/topic.rb

@@ -17,6 +17,7 @@ module Karafka
         max_messages
         max_wait_time
         initial_offset
+        consumer_persistence
       ].freeze
 
       private_constant :INHERITABLE_ATTRIBUTES
@@ -50,7 +51,7 @@ module Karafka
 
       # @return [Class] consumer class that we should use
       def consumer
-        if Karafka::App.config.consumer_persistence
+        if consumer_persistence
           # When persistence of consumers is on, no need to reload them
           @consumer
         else

data/lib/karafka/version.rb

@@ -3,5 +3,5 @@
 # Main module namespace
 module Karafka
   # Current Karafka version
-  VERSION = '2.1.7'
+  VERSION = '2.1.8'
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: karafka
 version: !ruby/object:Gem::Version
-  version: 2.1.7
+  version: 2.1.8
 platform: ruby
 authors:
 - Maciej Mensfeld
@@ -35,7 +35,7 @@ cert_chain:
   Qf04B9ceLUaC4fPVEz10FyobjaFoY4i32xRto3XnrzeAgfEe4swLq8bQsR3w/EF3
   MGU0FeSV2Yj7Xc2x/7BzLK8xQn5l7Yy75iPF+KP3vVmDHnNl
   -----END CERTIFICATE-----
-date: 2023-07-22 00:00:00.000000000 Z
+date: 2023-07-29 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: karafka-core

metadata.gz.sig

@@ -1,6 +1 @@
-u��o���ڏIo���l��*�mӝ)VӣSk���,e�L5ְ�d�`�<����8�
胁��yb���]b
-V���� &	�'˜)��O��9zނ{<
-[�:��ZA�嵘P@'�t��G�fc�ҍ���麍��B�e-Ծ�x�F}�Wn������q�?�
-�&�/���|'*�������Vc�	�/) ��	���h�R�
-���\xl�C?H��9m���	��g|���>)c�+�z���i7���*��Ҟ�":͍�O7P��e'�������V��
-� ��Lhav���
+�߼$tJt�po�n��K����>���9mK�*��=��pc�3��;����;zP���{ȁ½������Iԋ���ɹX*�����:`��	�>]�I׉��GA���w
N�^E���oZO��M�͔�+�`jlX����Z^ֿ��S�S��\ꡛ��–��-�f�HU@����1� WSn�H.�KfN��[������v+��Xt����A:��lO0hy��x3�]?�|(���
k��C+�	Xf���B�١����L!��}�(@tl��G����>)ߙ������Rxܦ�)5E�>�&-�
Z���l����ç���<��@cػM;�?ۗ��郆p)6��N��\�6ÄD�C��G?��~c��%