karafka 2.1.4 → 2.1.6

data/lib/karafka/processing/strategies/default.rb

@@ -27,12 +27,7 @@ module Karafka
           # Ignore earlier offsets than the one we already committed
           return true if coordinator.seek_offset > message.offset
           return false if revoked?
-
-          unless client.mark_as_consumed(message)
-            coordinator.revoke
-
-            return false
-          end
+          return revoked? unless client.mark_as_consumed(message)
 
           coordinator.seek_offset = message.offset + 1
 
@@ -49,17 +44,38 @@ module Karafka
           return true if coordinator.seek_offset > message.offset
           return false if revoked?
 
-          unless client.mark_as_consumed!(message)
-            coordinator.revoke
-
-            return false
-          end
+          return revoked? unless client.mark_as_consumed!(message)
 
           coordinator.seek_offset = message.offset + 1
 
           true
         end
 
+        # Triggers an async offset commit
+        #
+        # @param async [Boolean] should we use async (default) or sync commit
+        # @return [Boolean] true if we still own the partition.
+        # @note Due to its async nature, this may not fully represent the offset state in some
+        #   edge cases (like for example going beyond max.poll.interval)
+        def commit_offsets(async: true)
+          # Do not commit if we already lost the assignment
+          return false if revoked?
+          return true if client.commit_offsets(async: async)
+
+          # This will once more check the librdkafka revocation status and will revoke the
+          # coordinator in case it was not revoked
+          revoked?
+        end
+
+        # Triggers a synchronous offsets commit to Kafka
+        #
+        # @return [Boolean] true if we still own the partition, false otherwise.
+        # @note This is fully synchronous, hence the result of this can be used in DB transactions
+        #   etc as a way of making sure, that we still own the partition.
+        def commit_offsets!
+          commit_offsets(async: false)
+        end
+
         # No actions needed for the standard flow here
         def handle_before_enqueue
           nil

data/lib/karafka/railtie.rb

@@ -70,11 +70,11 @@ if Karafka.rails?
         # We can have many listeners, but it does not matter in which we will reload the code
         # as long as all the consumers will be re-created as Rails reload is thread-safe
         ::Karafka::App.monitor.subscribe('connection.listener.fetch_loop') do
-          # Reload code each time there is a change in the code
-          next unless Rails.application.reloaders.any?(&:updated?)
           # If consumer persistence is enabled, no reason to reload because we will still keep
           # old consumer instances in memory.
           next if Karafka::App.config.consumer_persistence
+          # Reload code each time there is a change in the code
+          next unless Rails.application.reloaders.any?(&:updated?)
 
           Rails.application.reloader.reload!
         end

data/lib/karafka/setup/attributes_map.rb

@@ -47,6 +47,7 @@ module Karafka
         fetch.max.bytes
         fetch.message.max.bytes
         fetch.min.bytes
+        fetch.queue.backoff.ms
         fetch.wait.max.ms
         group.id
         group.instance.id

data/lib/karafka/version.rb

@@ -3,5 +3,5 @@
 # Main module namespace
 module Karafka
   # Current Karafka version
-  VERSION = '2.1.4'
+  VERSION = '2.1.6'
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: karafka
 version: !ruby/object:Gem::Version
-  version: 2.1.4
+  version: 2.1.6
 platform: ruby
 authors:
 - Maciej Mensfeld
@@ -35,7 +35,7 @@ cert_chain:
   Qf04B9ceLUaC4fPVEz10FyobjaFoY4i32xRto3XnrzeAgfEe4swLq8bQsR3w/EF3
   MGU0FeSV2Yj7Xc2x/7BzLK8xQn5l7Yy75iPF+KP3vVmDHnNl
   -----END CERTIFICATE-----
-date: 2023-06-06 00:00:00.000000000 Z
+date: 2023-06-29 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: karafka-core
@@ -43,20 +43,20 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 2.0.13
+        version: 2.1.1
     - - "<"
       - !ruby/object:Gem::Version
-        version: 3.0.0
+        version: 2.2.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 2.0.13
+        version: 2.1.1
     - - "<"
       - !ruby/object:Gem::Version
-        version: 3.0.0
+        version: 2.2.0
 - !ruby/object:Gem::Dependency
   name: thor
   requirement: !ruby/object:Gem::Requirement
@@ -77,7 +77,7 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 2.5.3
+        version: 2.6.2
     - - "<"
       - !ruby/object:Gem::Version
         version: 3.0.0
@@ -87,7 +87,7 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 2.5.3
+        version: 2.6.2
     - - "<"
       - !ruby/object:Gem::Version
         version: 3.0.0
@@ -214,7 +214,6 @@ files:
 - lib/karafka/messages/parser.rb
 - lib/karafka/messages/seek.rb
 - lib/karafka/patches/rdkafka/bindings.rb
-- lib/karafka/patches/rdkafka/consumer.rb
 - lib/karafka/pro.rb
 - lib/karafka/pro/active_job/consumer.rb
 - lib/karafka/pro/active_job/dispatcher.rb
@@ -227,6 +226,8 @@ files:
 - lib/karafka/pro/encryption/messages/parser.rb
 - lib/karafka/pro/encryption/setup/config.rb
 - lib/karafka/pro/iterator.rb
+- lib/karafka/pro/iterator/expander.rb
+- lib/karafka/pro/iterator/tpl_builder.rb
 - lib/karafka/pro/loader.rb
 - lib/karafka/pro/performance_tracker.rb
 - lib/karafka/pro/processing/collapser.rb

data/lib/karafka/patches/rdkafka/consumer.rb

@@ -1,22 +0,0 @@
-# frozen_string_literal: true
-
-module Karafka
-  # Patches to external components
-  module Patches
-    # Rdkafka related patches
-    module Rdkafka
-      # Rdkafka::Consumer patches
-      module Consumer
-        # A method that allows us to get the native kafka producer name
-        # @return [String] producer instance name
-        # @note We need this to make sure that we allocate proper dispatched events only to
-        #   callback listeners that should publish them
-        def name
-          @name ||= ::Rdkafka::Bindings.rd_kafka_name(@native_kafka)
-        end
-      end
-    end
-  end
-end
-
-::Rdkafka::Consumer.include ::Karafka::Patches::Rdkafka::Consumer