karafka 2.0.11 → 2.0.12

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 00fdec9b351cb2da897d141a0dbb97816d7b0e53d7548b86d1ff48c5a8d5af3a
-  data.tar.gz: 514964e409f4904932ee9e7cc1aa27094820d2cbe875c75ad49c82ce41b8afbf
+  metadata.gz: 2c407cd113e41314102fc910cf0b35e8081c55bdadae66635afe59491b58e390
+  data.tar.gz: 336dad28cc65218e33a2b3bc42b69a166ff4e8e6a10c65a435e57f75eca5ac90
 SHA512:
-  metadata.gz: ba9281c9bfdb3a7a2a4c4121c26d1debb4bd39af07a190172a15f34705ee65bd9a1c8ce2e3773ccb3fe749dc83025bf96e0dde31664f1c6541e5923cad87b7a5
-  data.tar.gz: 95f150e80a77f68c16f0ecce849403a7afdcaf50f67bdaa1471074173f8b27b5a6b66d8a5cda8c3dc39106d8153ba843a35c9123883dc803b7bb05ec2c1f47c3
+  metadata.gz: 82ffee28acdaa1d126944426a7b31628380fc04aca24002ea55067f6c80a0b72a885d246a656913ea35e79fa82a9d741c35125e31b75abbbf7fb0022360e3b2b
+  data.tar.gz: b64f9dc4fb2461f3cbe3a774231a661365af5b1be88a6c95b1543521d19d8171514eb5111f88ceb53e309b5ab433f982b9535c2bbb4919e4c07850570424605a

data/CHANGELOG.md

@@ -1,5 +1,11 @@
 # Karafka framework changelog
 
+## 2.0.12 (2022-10-06)
+- Commit stored offsets upon rebalance revocation event to reduce number of messages that are re-processed.
+- Support cooperative-sticky rebalance strategy.
+- Replace offset commit after each batch with a per-rebalance commit.
+- User instrumentation to publish internal rebalance errors.
+
 ## 2.0.11 (2022-09-29)
 - Report early on errors related to network and on max poll interval being exceeded to indicate critical problems that will be retries but may mean some underlying problems in the system.
 - Fix support of Ruby 2.7.0 to 2.7.2 (#1045)

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    karafka (2.0.11)
+    karafka (2.0.12)
       karafka-core (>= 2.0.2, < 3.0.0)
       rdkafka (>= 0.12)
       thor (>= 0.20)
@@ -65,7 +65,7 @@ GEM
       karafka-core (>= 2.0.2, < 3.0.0)
       rdkafka (>= 0.10)
       zeitwerk (~> 2.3)
-    zeitwerk (2.6.0)
+    zeitwerk (2.6.1)
 
 PLATFORMS
   x86_64-linux

data/lib/karafka/connection/listener.rb

@@ -118,10 +118,6 @@ module Karafka
           build_and_schedule_consumption_jobs
 
           wait
-
-          # We don't use the `#commit_offsets!` here for performance reasons. This can be achieved
-          # if needed by using manual offset management.
-          @client.commit_offsets
         end
 
         # If we are stopping we will no longer schedule any jobs despite polling.

data/lib/karafka/connection/rebalance_manager.rb

@@ -14,6 +14,9 @@ module Karafka
     #   that are lost, are those that got revoked but did not get re-assigned back. We do not
     #   expose this concept outside and we normalize to have them revoked, as it is irrelevant
     #   from the rest of the code perspective as only those that are lost are truly revoked.
+    #
+    # @note For cooperative-sticky `#assigned_partitions` holds only the recently assigned
+    #   partitions, not all the partitions that are owned
     class RebalanceManager
       # Empty array for internal usage not to create new objects
       EMPTY_ARRAY = [].freeze

data/lib/karafka/patches/rdkafka/bindings.rb

@@ -0,0 +1,132 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Patches
+    module Rdkafka
+      # Binding patches that slightly change how rdkafka operates in certain places
+      module Bindings
+        include ::Rdkafka::Bindings
+
+        # Alias internally
+        RB = ::Rdkafka::Bindings
+
+        class << self
+          # Handle assignments on cooperative rebalance
+          #
+          # @param client_ptr [FFI::Pointer]
+          # @param code [Integer]
+          # @param partitions_ptr [FFI::Pointer]
+          def on_cooperative_rebalance(client_ptr, code, partitions_ptr)
+            case code
+            when RB::RD_KAFKA_RESP_ERR__ASSIGN_PARTITIONS
+              RB.rd_kafka_incremental_assign(client_ptr, partitions_ptr)
+            when RB::RD_KAFKA_RESP_ERR__REVOKE_PARTITIONS
+              RB.rd_kafka_commit(client_ptr, nil, false)
+              RB.rd_kafka_incremental_unassign(client_ptr, partitions_ptr)
+            else
+              RB.rd_kafka_assign(client_ptr, FFI::Pointer::NULL)
+            end
+          end
+
+          # Handle assignments on a eager rebalance
+          #
+          # @param client_ptr [FFI::Pointer]
+          # @param code [Integer]
+          # @param partitions_ptr [FFI::Pointer]
+          def on_eager_rebalance(client_ptr, code, partitions_ptr)
+            case code
+            when RB::RD_KAFKA_RESP_ERR__ASSIGN_PARTITIONS
+              RB.rd_kafka_assign(client_ptr, partitions_ptr)
+            when RB::RD_KAFKA_RESP_ERR__REVOKE_PARTITIONS
+              RB.rd_kafka_commit(client_ptr, nil, false)
+              RB.rd_kafka_assign(client_ptr, FFI::Pointer::NULL)
+            else
+              RB.rd_kafka_assign(client_ptr, FFI::Pointer::NULL)
+            end
+          end
+
+          # Trigger Karafka callbacks
+          #
+          # @param code [Integer]
+          # @param opaque [Rdkafka::Opaque]
+          # @param consumer [Rdkafka::Consumer]
+          # @param tpl [Rdkafka::Consumer::TopicPartitionList]
+          def trigger_callbacks(code, opaque, consumer, tpl)
+            case code
+            when RB::RD_KAFKA_RESP_ERR__ASSIGN_PARTITIONS
+              opaque.call_on_partitions_assigned(consumer, tpl)
+            when RB::RD_KAFKA_RESP_ERR__REVOKE_PARTITIONS
+              opaque.call_on_partitions_revoked(consumer, tpl)
+            end
+          rescue StandardError => e
+            Karafka.monitor.instrument(
+              'error.occurred',
+              caller: self,
+              error: e,
+              type: 'connection.client.rebalance_callback.error'
+            )
+          end
+        end
+
+        # This patch changes few things:
+        # - it commits offsets (if any) upon partition revocation, so less jobs need to be
+        #   reprocessed if they are assigned to a different process
+        # - reports callback errors into the errors instrumentation instead of the logger
+        # - catches only StandardError instead of Exception as we fully control the directly
+        #   executed callbacks
+        #
+        # @see https://docs.confluent.io/2.0.0/clients/librdkafka/classRdKafka_1_1RebalanceCb.html
+        RebalanceCallback = FFI::Function.new(
+          :void, %i[pointer int pointer pointer]
+        ) do |client_ptr, code, partitions_ptr, opaque_ptr|
+          # Patch reference
+          pr = ::Karafka::Patches::Rdkafka::Bindings
+
+          if RB.rd_kafka_rebalance_protocol(client_ptr) == 'COOPERATIVE'
+            pr.on_cooperative_rebalance(client_ptr, code, partitions_ptr)
+          else
+            pr.on_eager_rebalance(client_ptr, code, partitions_ptr)
+          end
+
+          opaque = ::Rdkafka::Config.opaques[opaque_ptr.to_i]
+          return unless opaque
+
+          tpl = ::Rdkafka::Consumer::TopicPartitionList.from_native_tpl(partitions_ptr).freeze
+          consumer = ::Rdkafka::Consumer.new(client_ptr)
+
+          pr.trigger_callbacks(code, opaque, consumer, tpl)
+        end
+      end
+    end
+  end
+end
+
+# We need to replace the original callback with ours.
+# At the moment there is no API in rdkafka-ruby to do so
+::Rdkafka::Bindings.send(
+  :remove_const,
+  'RebalanceCallback'
+)
+
+::Rdkafka::Bindings.const_set(
+  'RebalanceCallback',
+  Karafka::Patches::Rdkafka::Bindings::RebalanceCallback
+)
+
+::Rdkafka::Bindings.attach_function(
+  :rd_kafka_rebalance_protocol,
+  %i[pointer],
+  :string
+)
+
+::Rdkafka::Bindings.attach_function(
+  :rd_kafka_incremental_assign,
+  %i[pointer pointer],
+  :string
+)
+
+::Rdkafka::Bindings.attach_function(
+  :rd_kafka_incremental_unassign,
+  %i[pointer pointer],
+  :string
+)

data/lib/karafka/version.rb

@@ -3,5 +3,5 @@
 # Main module namespace
 module Karafka
   # Current Karafka version
-  VERSION = '2.0.11'
+  VERSION = '2.0.12'
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: karafka
 version: !ruby/object:Gem::Version
-  version: 2.0.11
+  version: 2.0.12
 platform: ruby
 authors:
 - Maciej Mensfeld
@@ -35,7 +35,7 @@ cert_chain:
   Qf04B9ceLUaC4fPVEz10FyobjaFoY4i32xRto3XnrzeAgfEe4swLq8bQsR3w/EF3
   MGU0FeSV2Yj7Xc2x/7BzLK8xQn5l7Yy75iPF+KP3vVmDHnNl
   -----END CERTIFICATE-----
-date: 2022-09-29 00:00:00.000000000 Z
+date: 2022-10-06 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: karafka-core
@@ -212,6 +212,7 @@ files:
 - lib/karafka/messages/messages.rb
 - lib/karafka/messages/metadata.rb
 - lib/karafka/messages/seek.rb
+- lib/karafka/patches/rdkafka/bindings.rb
 - lib/karafka/patches/rdkafka/consumer.rb
 - lib/karafka/pro.rb
 - lib/karafka/pro/active_job/consumer.rb

metadata.gz.sig

@@ -1,3 +1,2 @@
-a��Z���hR�h)�/Q��(ΊW�y�F�
-H]�mm�(����� C��#�����NJ^)dP���W��q��Y���N��d���	031x?�yb���F��_ܥ�#p|�gοv��Jݤa ��h_���T��V��5v�/����5-�I�M�fH���Әm��)�?�4@	���c	�����L��wg_e�ꁬ���33����39�����@�u��/��P�u�U�(d�0а�ZI�Y1 -�[��͸��hr�j������e2��q{��9������[�Y�)�2�d�wLu��9�G�
-�L���9V����~6�?�M�X�S0�~{�2�r%�j�
�dO_�zK5x�	���l�D������e+�:�q�%���
+�
JMT
+6���]�������˪�`�m/��Sݔ1`��#7�@���8�x7|��d��W�pчH�k.�9�<