karafka 2.1.5.beta1 → 2.1.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 967412d28c31483df6a0c151ec0578367da24e503184608789791774347ffc53
-  data.tar.gz: '0828ba1db27f9f287b3ab9f80928816a81ae45ad79ce092e02cccd33b9bac204'
+  metadata.gz: 82a8b28b55f1db0808be3d1e48616f6b2389466332c9116e263e37cab992fc65
+  data.tar.gz: 2f29bb9bb1c3f949d206c5c8453b35ad163219babb48687e2270e13914e78aba
 SHA512:
-  metadata.gz: 3774daedd74efe8e2498e97e503226cc01848e7a7be6ca255852bcaf112bc790ff98953584c771de8afa96142e8244e9f1b036b18490305766b428671b136bce
-  data.tar.gz: b575cf2be3d4cdcb73fec9cdfd65cd705408d1efae39ed200ae2d815e7c9241f582d5ebb8b802347289de910ee50589f0551bb4c194a161905e38425d05bd296
+  metadata.gz: 93a66f4aeb49cea810bfd90cf424b3334d1dae992035e0bd9613bbd3c42f642f94fd0efd979d57df5083a46f66f522a7d3952c9e24340b8a4dc4c23aff165a0f
+  data.tar.gz: 4ee03b442b3029aecf0ffd636ddccb054e51f2a448c3dd642993464bfc32aa45595f26835db8a9b5b01940ab5b532e0bc22a9a3cdbcc9899320b55010473c749

data/CHANGELOG.md

@@ -1,11 +1,13 @@
 # Karafka framework changelog
 
-## 2.1.5 (Unreleased)
+## 2.1.5 (2023-06-19)
 - [Improvement] Drastically improve `#revoked?` response quality by checking the real time assignment lost state on librdkafka.
 - [Improvement] Improve eviction of saturated jobs that would run on already revoked assignments.
 - [Improvement] Expose `#commit_offsets` and `#commit_offsets!` methods in the consumer to provide ability to commit offsets directly to Kafka without having to mark new messages as consumed.
 - [Improvement] No longer skip offset commit when no messages marked as consumed as `librdkafka` has fixed the crashes there.
 - [Improvement] Remove no longer needed patches.
+- [Improvement] Ensure, that the coordinator revocation status is switched upon revocation detection when using `#revoked?`
+- [Improvement] Add benchmarks for marking as consumed (sync and async).
 - [Change] Require `karafka-core` `>= 2.1.0`
 - [Change] Require `waterdrop` `>= 2.6.1`
 

data/Gemfile.lock

@@ -1,10 +1,10 @@
 PATH
   remote: .
   specs:
-    karafka (2.1.5.beta1)
-      karafka-core (>= 2.1.0.beta1, < 2.2.0)
+    karafka (2.1.5)
+      karafka-core (>= 2.1.0, < 2.2.0)
       thor (>= 0.20)
-      waterdrop (>= 2.6.1.beta1, < 3.0.0)
+      waterdrop (>= 2.6.1, < 3.0.0)
       zeitwerk (~> 2.3)
 
 GEM
@@ -30,10 +30,10 @@ GEM
       activesupport (>= 5.0)
     i18n (1.14.1)
       concurrent-ruby (~> 1.0)
-    karafka-core (2.1.0.beta1)
+    karafka-core (2.1.0)
       concurrent-ruby (>= 1.1)
-      karafka-rdkafka (>= 0.13.0.beta2, < 0.14.0)
-    karafka-rdkafka (0.13.0.beta2)
+      karafka-rdkafka (>= 0.13.0, < 0.14.0)
+    karafka-rdkafka (0.13.0)
       ffi (~> 1.15)
       mini_portile2 (~> 2.6)
       rake (> 12)
@@ -72,8 +72,8 @@ GEM
     tilt (2.2.0)
     tzinfo (2.0.6)
       concurrent-ruby (~> 1.0)
-    waterdrop (2.6.1.beta1)
-      karafka-core (>= 2.1.0.beta1, < 3.0.0)
+    waterdrop (2.6.1)
+      karafka-core (>= 2.1.0, < 3.0.0)
       zeitwerk (~> 2.3)
     zeitwerk (2.6.8)
 

data/karafka.gemspec

@@ -21,9 +21,9 @@ Gem::Specification.new do |spec|
     without having to focus on things that are not your business domain.
   DESC
 
-  spec.add_dependency 'karafka-core', '>= 2.1.0.beta1', '< 2.2.0'
+  spec.add_dependency 'karafka-core', '>= 2.1.0', '< 2.2.0'
   spec.add_dependency 'thor', '>= 0.20'
-  spec.add_dependency 'waterdrop', '>= 2.6.1.beta1', '< 3.0.0'
+  spec.add_dependency 'waterdrop', '>= 2.6.1', '< 3.0.0'
   spec.add_dependency 'zeitwerk', '~> 2.3'
 
   if $PROGRAM_NAME.end_with?('gem')

data/lib/karafka/base_consumer.rb

@@ -221,7 +221,12 @@ module Karafka
     #   even before we poll but it gets reset when polling happens, hence we also need to switch
     #   the coordinator state after the revocation (but prior to running more jobs)
     def revoked?
-      client.assignment_lost? || coordinator.revoked?
+      return true if coordinator.revoked?
+      return false unless client.assignment_lost?
+
+      coordinator.revoke
+
+      true
     end
 
     # @return [Boolean] are we retrying processing after an error. This can be used to provide a

data/lib/karafka/connection/client.rb

@@ -122,6 +122,10 @@ module Karafka
       # @note This will commit all the offsets for the whole consumer. In order to achieve
       #   granular control over where the offset should be for particular topic partitions, the
       #   store_offset should be used to only store new offset when we want them to be flushed
+      #
+      # @note This method for async may return `true` despite involuntary partition revocation as
+      #   it does **not** resolve to `lost_assignment?`. It returns only the commit state operation
+      #   result.
       def commit_offsets(async: true)
         @mutex.lock
 
@@ -238,9 +242,10 @@ module Karafka
       # @param [Karafka::Messages::Message] message that we want to mark as processed
       # @return [Boolean] true if successful. False if we no longer own given partition
       # @note This method won't trigger automatic offsets commits, rather relying on the offset
-      #   check-pointing trigger that happens with each batch processed
+      #   check-pointing trigger that happens with each batch processed. It will however check the
+      #   `librdkafka` assignment ownership to increase accuracy for involuntary revocations.
       def mark_as_consumed(message)
-        store_offset(message)
+        store_offset(message) && !assignment_lost?
       end
 
       # Marks a given message as consumed and commits the offsets in a blocking way.

data/lib/karafka/processing/strategies/default.rb

@@ -27,12 +27,7 @@ module Karafka
           # Ignore earlier offsets than the one we already committed
           return true if coordinator.seek_offset > message.offset
           return false if revoked?
-
-          unless client.mark_as_consumed(message)
-            coordinator.revoke
-
-            return false
-          end
+          return revoked? unless client.mark_as_consumed(message)
 
           coordinator.seek_offset = message.offset + 1
 
@@ -49,11 +44,7 @@ module Karafka
           return true if coordinator.seek_offset > message.offset
           return false if revoked?
 
-          unless client.mark_as_consumed!(message)
-            coordinator.revoke
-
-            return false
-          end
+          return revoked? unless client.mark_as_consumed!(message)
 
           coordinator.seek_offset = message.offset + 1
 
@@ -62,11 +53,18 @@ module Karafka
 
         # Triggers an async offset commit
         #
+        # @param async [Boolean] should we use async (default) or sync commit
         # @return [Boolean] true if we still own the partition.
         # @note Due to its async nature, this may not fully represent the offset state in some
         #   edge cases (like for example going beyond max.poll.interval)
-        def commit_offsets
-          client.commit_offsets(async: true)
+        def commit_offsets(async: true)
+          # Do not commit if we already lost the assignment
+          return false if revoked?
+          return true if client.commit_offsets(async: async)
+
+          # This will once more check the librdkafka revocation status and will revoke the
+          # coordinator in case it was not revoked
+          revoked?
         end
 
         # Triggers a synchronous offsets commit to Kafka
@@ -75,7 +73,7 @@ module Karafka
         # @note This is fully synchronous, hence the result of this can be used in DB transactions
         #   etc as a way of making sure, that we still own the partition.
         def commit_offsets!
-          client.commit_offsets(async: false)
+          commit_offsets(async: false)
         end
 
         # No actions needed for the standard flow here

data/lib/karafka/version.rb

@@ -3,5 +3,5 @@
 # Main module namespace
 module Karafka
   # Current Karafka version
-  VERSION = '2.1.5.beta1'
+  VERSION = '2.1.5'
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: karafka
 version: !ruby/object:Gem::Version
-  version: 2.1.5.beta1
+  version: 2.1.5
 platform: ruby
 authors:
 - Maciej Mensfeld
@@ -35,7 +35,7 @@ cert_chain:
   Qf04B9ceLUaC4fPVEz10FyobjaFoY4i32xRto3XnrzeAgfEe4swLq8bQsR3w/EF3
   MGU0FeSV2Yj7Xc2x/7BzLK8xQn5l7Yy75iPF+KP3vVmDHnNl
   -----END CERTIFICATE-----
-date: 2023-06-18 00:00:00.000000000 Z
+date: 2023-06-19 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: karafka-core
@@ -43,7 +43,7 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 2.1.0.beta1
+        version: 2.1.0
     - - "<"
       - !ruby/object:Gem::Version
         version: 2.2.0
@@ -53,7 +53,7 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 2.1.0.beta1
+        version: 2.1.0
     - - "<"
       - !ruby/object:Gem::Version
         version: 2.2.0
@@ -77,7 +77,7 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 2.6.1.beta1
+        version: 2.6.1
     - - "<"
       - !ruby/object:Gem::Version
         version: 3.0.0
@@ -87,7 +87,7 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 2.6.1.beta1
+        version: 2.6.1
     - - "<"
       - !ruby/object:Gem::Version
         version: 3.0.0
@@ -414,9 +414,9 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ">"
+  - - ">="
     - !ruby/object:Gem::Version
-      version: 1.3.1
+      version: '0'
 requirements: []
 rubygems_version: 3.4.10
 signing_key: