karafka 2.1.4 → 2.1.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: e54dc80fd5a3a93857fd6a3a175030f7cb2eee1f77668d6936b705b0789c4228
-  data.tar.gz: e7f3b02811432987048bc599c2c4a6f0444fe13d98ca4aa8e87cebd5aaa8037a
+  metadata.gz: 82a8b28b55f1db0808be3d1e48616f6b2389466332c9116e263e37cab992fc65
+  data.tar.gz: 2f29bb9bb1c3f949d206c5c8453b35ad163219babb48687e2270e13914e78aba
 SHA512:
-  metadata.gz: 871ed7a55421041b1b232c47b3da0f625b8f2114c00057daf883223d3da4dc16637c9f0b4714cebe4a5f33249beac9bd7abbe15ec122f5b1a45a3c953dcb1465
-  data.tar.gz: ac2cc50c3277bd0bac7e3a2af40cc6d5a9c3aa92fc8cc3d2b12b8645329d269d0ed0d80c462d2fa6a03a677213e84802a1dfe8fdc1c494348b1a8b4fd9ef8459
+  metadata.gz: 93a66f4aeb49cea810bfd90cf424b3334d1dae992035e0bd9613bbd3c42f642f94fd0efd979d57df5083a46f66f522a7d3952c9e24340b8a4dc4c23aff165a0f
+  data.tar.gz: 4ee03b442b3029aecf0ffd636ddccb054e51f2a448c3dd642993464bfc32aa45595f26835db8a9b5b01940ab5b532e0bc22a9a3cdbcc9899320b55010473c749

data/CHANGELOG.md

@@ -1,5 +1,16 @@
 # Karafka framework changelog
 
+## 2.1.5 (2023-06-19)
+- [Improvement] Drastically improve `#revoked?` response quality by checking the real time assignment lost state on librdkafka.
+- [Improvement] Improve eviction of saturated jobs that would run on already revoked assignments.
+- [Improvement] Expose `#commit_offsets` and `#commit_offsets!` methods in the consumer to provide ability to commit offsets directly to Kafka without having to mark new messages as consumed.
+- [Improvement] No longer skip offset commit when no messages marked as consumed as `librdkafka` has fixed the crashes there.
+- [Improvement] Remove no longer needed patches.
+- [Improvement] Ensure, that the coordinator revocation status is switched upon revocation detection when using `#revoked?`
+- [Improvement] Add benchmarks for marking as consumed (sync and async).
+- [Change] Require `karafka-core` `>= 2.1.0`
+- [Change] Require `waterdrop` `>= 2.6.1`
+
 ## 2.1.4 (2023-06-06)
 - [Fix] `processing_lag` and `consumption_lag` on empty batch fail on shutdown usage (#1475)
 

data/Gemfile.lock

@@ -1,19 +1,19 @@
 PATH
   remote: .
   specs:
-    karafka (2.1.4)
-      karafka-core (>= 2.0.13, < 3.0.0)
+    karafka (2.1.5)
+      karafka-core (>= 2.1.0, < 2.2.0)
       thor (>= 0.20)
-      waterdrop (>= 2.5.3, < 3.0.0)
+      waterdrop (>= 2.6.1, < 3.0.0)
       zeitwerk (~> 2.3)
 
 GEM
   remote: https://rubygems.org/
   specs:
-    activejob (7.0.4.3)
-      activesupport (= 7.0.4.3)
+    activejob (7.0.5)
+      activesupport (= 7.0.5)
       globalid (>= 0.3.6)
-    activesupport (7.0.4.3)
+    activesupport (7.0.5)
       concurrent-ruby (~> 1.0, >= 1.0.2)
       i18n (>= 1.6, < 2)
       minitest (>= 5.1)
@@ -28,26 +28,26 @@ GEM
     ffi (1.15.5)
     globalid (1.1.0)
       activesupport (>= 5.0)
-    i18n (1.13.0)
+    i18n (1.14.1)
       concurrent-ruby (~> 1.0)
-    karafka-core (2.0.13)
+    karafka-core (2.1.0)
       concurrent-ruby (>= 1.1)
-      karafka-rdkafka (>= 0.12.3)
-    karafka-rdkafka (0.12.3)
+      karafka-rdkafka (>= 0.13.0, < 0.14.0)
+    karafka-rdkafka (0.13.0)
       ffi (~> 1.15)
       mini_portile2 (~> 2.6)
       rake (> 12)
-    karafka-web (0.5.2)
+    karafka-web (0.6.0)
       erubi (~> 1.4)
-      karafka (>= 2.0.40, < 3.0.0)
-      karafka-core (>= 2.0.12, < 3.0.0)
-      roda (~> 3.63)
+      karafka (>= 2.1.4, < 3.0.0)
+      karafka-core (>= 2.0.13, < 3.0.0)
+      roda (~> 3.68, >= 3.68)
       tilt (~> 2.0)
     mini_portile2 (2.8.2)
-    minitest (5.18.0)
-    rack (3.0.7)
+    minitest (5.18.1)
+    rack (3.0.8)
     rake (13.0.6)
-    roda (3.68.0)
+    roda (3.69.0)
       rack
     rspec (3.12.0)
       rspec-core (~> 3.12.0)
@@ -69,11 +69,11 @@ GEM
     simplecov-html (0.12.3)
     simplecov_json_formatter (0.1.4)
     thor (1.2.2)
-    tilt (2.1.0)
+    tilt (2.2.0)
     tzinfo (2.0.6)
       concurrent-ruby (~> 1.0)
-    waterdrop (2.5.3)
-      karafka-core (>= 2.0.13, < 3.0.0)
+    waterdrop (2.6.1)
+      karafka-core (>= 2.1.0, < 3.0.0)
       zeitwerk (~> 2.3)
     zeitwerk (2.6.8)
 

data/karafka.gemspec

@@ -21,9 +21,9 @@ Gem::Specification.new do |spec|
     without having to focus on things that are not your business domain.
   DESC
 
-  spec.add_dependency 'karafka-core', '>= 2.0.13', '< 3.0.0'
+  spec.add_dependency 'karafka-core', '>= 2.1.0', '< 2.2.0'
   spec.add_dependency 'thor', '>= 0.20'
-  spec.add_dependency 'waterdrop', '>= 2.5.3', '< 3.0.0'
+  spec.add_dependency 'waterdrop', '>= 2.6.1', '< 3.0.0'
   spec.add_dependency 'zeitwerk', '~> 2.3'
 
   if $PROGRAM_NAME.end_with?('gem')

data/lib/karafka/admin.rb

@@ -171,7 +171,9 @@ module Karafka
       # @return [Rdkafka::Metadata] cluster metadata info
       def cluster_info
         with_admin do |admin|
-          Rdkafka::Metadata.new(admin.instance_variable_get('@native_kafka'))
+          admin.instance_variable_get('@native_kafka').with_inner do |inner|
+            Rdkafka::Metadata.new(inner)
+          end
         end
       end
 

data/lib/karafka/base_consumer.rb

@@ -215,10 +215,18 @@ module Karafka
     end
 
     # @return [Boolean] true if partition was revoked from the current consumer
-    # @note We know that partition got revoked because when we try to mark message as consumed,
-    #   unless if is successful, it will return false
+    # @note There are two "levels" on which we can know that partition was revoked. First one is
+    #   when we loose the assignment involuntarily and second is when coordinator gets this info
+    #   after we poll with the rebalance callbacks. The first check allows us to get this notion
+    #   even before we poll but it gets reset when polling happens, hence we also need to switch
+    #   the coordinator state after the revocation (but prior to running more jobs)
     def revoked?
-      coordinator.revoked?
+      return true if coordinator.revoked?
+      return false unless client.assignment_lost?
+
+      coordinator.revoke
+
+      true
     end
 
     # @return [Boolean] are we retrying processing after an error. This can be used to provide a

data/lib/karafka/connection/client.rb

@@ -41,9 +41,6 @@ module Karafka
         @buffer = RawMessagesBuffer.new
         @rebalance_manager = RebalanceManager.new
         @kafka = build_consumer
-        # Marks if we need to offset. If we did not store offsets, we should not commit the offset
-        # position as it will crash rdkafka
-        @offsetting = false
         # We need to keep track of what we have paused for resuming
         # In case we loose partition, we still need to resume it, otherwise it won't be fetched
         # again if we get reassigned to it later on. We need to keep them as after revocation we
@@ -109,8 +106,14 @@ module Karafka
         end
       end
 
+      # @return [Boolean] true if our current assignment has been lost involuntarily.
+      def assignment_lost?
+        @mutex.synchronize do
+          @kafka.assignment_lost?
+        end
+      end
+
       # Commits the offset on a current consumer in a non-blocking or blocking way.
-      # Ignoring a case where there would not be an offset (for example when rebalance occurs).
       #
       # @param async [Boolean] should the commit happen async or sync (async by default)
       # @return [Boolean] did committing was successful. It may be not, when we no longer own
@@ -118,7 +121,11 @@ module Karafka
       #
       # @note This will commit all the offsets for the whole consumer. In order to achieve
       #   granular control over where the offset should be for particular topic partitions, the
-      #   store_offset should be used to only store new offset when we want to to be flushed
+      #   store_offset should be used to only store new offset when we want them to be flushed
+      #
+      # @note This method for async may return `true` despite involuntary partition revocation as
+      #   it does **not** resolve to `lost_assignment?`. It returns only the commit state operation
+      #   result.
       def commit_offsets(async: true)
         @mutex.lock
 
@@ -235,9 +242,10 @@ module Karafka
       # @param [Karafka::Messages::Message] message that we want to mark as processed
       # @return [Boolean] true if successful. False if we no longer own given partition
       # @note This method won't trigger automatic offsets commits, rather relying on the offset
-      #   check-pointing trigger that happens with each batch processed
+      #   check-pointing trigger that happens with each batch processed. It will however check the
+      #   `librdkafka` assignment ownership to increase accuracy for involuntary revocations.
       def mark_as_consumed(message)
-        store_offset(message)
+        store_offset(message) && !assignment_lost?
       end
 
       # Marks a given message as consumed and commits the offsets in a blocking way.
@@ -256,7 +264,6 @@ module Karafka
 
         @mutex.synchronize do
           @closed = false
-          @offsetting = false
           @paused_tpls.clear
           @kafka = build_consumer
         end
@@ -281,7 +288,6 @@ module Karafka
       # @param message [Karafka::Messages::Message]
       # @return [Boolean] true if we could store the offset (if we still own the partition)
       def internal_store_offset(message)
-        @offsetting = true
         @kafka.store_offset(message)
         true
       rescue Rdkafka::RdkafkaError => e
@@ -294,11 +300,11 @@ module Karafka
       # Non thread-safe message committing method
       # @param async [Boolean] should the commit happen async or sync (async by default)
       # @return [Boolean] true if offset commit worked, false if we've lost the assignment
+      # @note We do **not** consider `no_offset` as any problem and we allow to commit offsets
+      #   even when no stored, because with sync commit, it refreshes the ownership state of the
+      #   consumer in a sync way.
       def internal_commit_offsets(async: true)
-        return true unless @offsetting
-
         @kafka.commit(nil, async)
-        @offsetting = false
 
         true
       rescue Rdkafka::RdkafkaError => e

data/lib/karafka/connection/rebalance_manager.rb

@@ -49,9 +49,8 @@ module Karafka
       # Callback that kicks in inside of rdkafka, when new partitions are assigned.
       #
       # @private
-      # @param _ [Rdkafka::Consumer]
       # @param partitions [Rdkafka::Consumer::TopicPartitionList]
-      def on_partitions_assigned(_, partitions)
+      def on_partitions_assigned(partitions)
         @assigned_partitions = partitions.to_h.transform_values { |part| part.map(&:partition) }
         @changed = true
       end
@@ -59,9 +58,8 @@ module Karafka
       # Callback that kicks in inside of rdkafka, when partitions are revoked.
       #
       # @private
-      # @param _ [Rdkafka::Consumer]
       # @param partitions [Rdkafka::Consumer::TopicPartitionList]
-      def on_partitions_revoked(_, partitions)
+      def on_partitions_revoked(partitions)
         @revoked_partitions = partitions.to_h.transform_values { |part| part.map(&:partition) }
         @changed = true
       end

data/lib/karafka/messages/builders/message.rb

@@ -12,9 +12,6 @@ module Karafka
           # @param received_at [Time] moment when we've received the message
           # @return [Karafka::Messages::Message] message object with payload and metadata
           def call(kafka_message, topic, received_at)
-            # @see https://github.com/appsignal/rdkafka-ruby/issues/168
-            kafka_message.headers.transform_keys!(&:to_s)
-
             metadata = Karafka::Messages::Metadata.new(
               timestamp: kafka_message.timestamp,
               headers: kafka_message.headers,

data/lib/karafka/patches/rdkafka/bindings.rb

@@ -49,14 +49,13 @@ module Karafka
           #
           # @param code [Integer]
           # @param opaque [Rdkafka::Opaque]
-          # @param consumer [Rdkafka::Consumer]
           # @param tpl [Rdkafka::Consumer::TopicPartitionList]
-          def trigger_callbacks(code, opaque, consumer, tpl)
+          def trigger_callbacks(code, opaque, tpl)
             case code
             when RB::RD_KAFKA_RESP_ERR__ASSIGN_PARTITIONS
-              opaque.call_on_partitions_assigned(consumer, tpl)
+              opaque.call_on_partitions_assigned(tpl)
             when RB::RD_KAFKA_RESP_ERR__REVOKE_PARTITIONS
-              opaque.call_on_partitions_revoked(consumer, tpl)
+              opaque.call_on_partitions_revoked(tpl)
             end
           rescue StandardError => e
             Karafka.monitor.instrument(
@@ -92,9 +91,8 @@ module Karafka
           return unless opaque
 
           tpl = ::Rdkafka::Consumer::TopicPartitionList.from_native_tpl(partitions_ptr).freeze
-          consumer = ::Rdkafka::Consumer.new(client_ptr)
 
-          pr.trigger_callbacks(code, opaque, consumer, tpl)
+          pr.trigger_callbacks(code, opaque, tpl)
         end
       end
     end

data/lib/karafka/processing/strategies/default.rb

@@ -27,12 +27,7 @@ module Karafka
           # Ignore earlier offsets than the one we already committed
           return true if coordinator.seek_offset > message.offset
           return false if revoked?
-
-          unless client.mark_as_consumed(message)
-            coordinator.revoke
-
-            return false
-          end
+          return revoked? unless client.mark_as_consumed(message)
 
           coordinator.seek_offset = message.offset + 1
 
@@ -49,17 +44,38 @@ module Karafka
           return true if coordinator.seek_offset > message.offset
           return false if revoked?
 
-          unless client.mark_as_consumed!(message)
-            coordinator.revoke
-
-            return false
-          end
+          return revoked? unless client.mark_as_consumed!(message)
 
           coordinator.seek_offset = message.offset + 1
 
           true
         end
 
+        # Triggers an async offset commit
+        #
+        # @param async [Boolean] should we use async (default) or sync commit
+        # @return [Boolean] true if we still own the partition.
+        # @note Due to its async nature, this may not fully represent the offset state in some
+        #   edge cases (like for example going beyond max.poll.interval)
+        def commit_offsets(async: true)
+          # Do not commit if we already lost the assignment
+          return false if revoked?
+          return true if client.commit_offsets(async: async)
+
+          # This will once more check the librdkafka revocation status and will revoke the
+          # coordinator in case it was not revoked
+          revoked?
+        end
+
+        # Triggers a synchronous offsets commit to Kafka
+        #
+        # @return [Boolean] true if we still own the partition, false otherwise.
+        # @note This is fully synchronous, hence the result of this can be used in DB transactions
+        #   etc as a way of making sure, that we still own the partition.
+        def commit_offsets!
+          commit_offsets(async: false)
+        end
+
         # No actions needed for the standard flow here
         def handle_before_enqueue
           nil

data/lib/karafka/setup/attributes_map.rb

@@ -47,6 +47,7 @@ module Karafka
         fetch.max.bytes
         fetch.message.max.bytes
         fetch.min.bytes
+        fetch.queue.backoff.ms
         fetch.wait.max.ms
         group.id
         group.instance.id

data/lib/karafka/version.rb

@@ -3,5 +3,5 @@
 # Main module namespace
 module Karafka
   # Current Karafka version
-  VERSION = '2.1.4'
+  VERSION = '2.1.5'
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: karafka
 version: !ruby/object:Gem::Version
-  version: 2.1.4
+  version: 2.1.5
 platform: ruby
 authors:
 - Maciej Mensfeld
@@ -35,7 +35,7 @@ cert_chain:
   Qf04B9ceLUaC4fPVEz10FyobjaFoY4i32xRto3XnrzeAgfEe4swLq8bQsR3w/EF3
   MGU0FeSV2Yj7Xc2x/7BzLK8xQn5l7Yy75iPF+KP3vVmDHnNl
   -----END CERTIFICATE-----
-date: 2023-06-06 00:00:00.000000000 Z
+date: 2023-06-19 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: karafka-core
@@ -43,20 +43,20 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 2.0.13
+        version: 2.1.0
     - - "<"
       - !ruby/object:Gem::Version
-        version: 3.0.0
+        version: 2.2.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 2.0.13
+        version: 2.1.0
     - - "<"
       - !ruby/object:Gem::Version
-        version: 3.0.0
+        version: 2.2.0
 - !ruby/object:Gem::Dependency
   name: thor
   requirement: !ruby/object:Gem::Requirement
@@ -77,7 +77,7 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 2.5.3
+        version: 2.6.1
     - - "<"
       - !ruby/object:Gem::Version
         version: 3.0.0
@@ -87,7 +87,7 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 2.5.3
+        version: 2.6.1
     - - "<"
       - !ruby/object:Gem::Version
         version: 3.0.0
@@ -214,7 +214,6 @@ files:
 - lib/karafka/messages/parser.rb
 - lib/karafka/messages/seek.rb
 - lib/karafka/patches/rdkafka/bindings.rb
-- lib/karafka/patches/rdkafka/consumer.rb
 - lib/karafka/pro.rb
 - lib/karafka/pro/active_job/consumer.rb
 - lib/karafka/pro/active_job/dispatcher.rb

metadata.gz.sig

@@ -1,7 +1 @@
-J�SR�3a����Ɍ�-7.���5k�H������"�\���7���h�����i��	Gj\���F�e)�5
-ޘVA
O�V�P"�i(�%v+F
-�#�
�RF��0�`���u�����3�
-�hL�@wVS���:�oY�'P�q�-��P&��R8r4��!����b�o=1�P��>�Q[�}&�U/i]�����-0�5����8@���w,}���o�G�Ǎ�֡�B���1Y ���s+i��J�R��w�e��-�"*0�7�X
-���L�hWy��*�
-͝飝�'���
�(R�{\�H]0ƴ|ݖ��P)���C�
-�����JB]�
e
�)M�:6
+���B֣ުJ=1��|�J/m^�u��ˤ�^�qS2�/���e4��4t.�T�<$�H���U�p.����~�R�֪�VY^U�7���v`�I

data/lib/karafka/patches/rdkafka/consumer.rb

@@ -1,22 +0,0 @@
-# frozen_string_literal: true
-
-module Karafka
-  # Patches to external components
-  module Patches
-    # Rdkafka related patches
-    module Rdkafka
-      # Rdkafka::Consumer patches
-      module Consumer
-        # A method that allows us to get the native kafka producer name
-        # @return [String] producer instance name
-        # @note We need this to make sure that we allocate proper dispatched events only to
-        #   callback listeners that should publish them
-        def name
-          @name ||= ::Rdkafka::Bindings.rd_kafka_name(@native_kafka)
-        end
-      end
-    end
-  end
-end
-
-::Rdkafka::Consumer.include ::Karafka::Patches::Rdkafka::Consumer