karafka 2.4.16 → 2.4.18

data/lib/karafka/pro/processing/strategies/aj/dlq_ftr_lrj_mom_vp.rb

@@ -1,15 +1,7 @@
 # frozen_string_literal: true
 
-# This Karafka component is a Pro component under a commercial license.
-# This Karafka component is NOT licensed under LGPL.
-#
-# All of the commercial components are present in the lib/karafka/pro directory of this
-# repository and their usage requires commercial license agreement.
-#
-# Karafka has also commercial-friendly license, commercial support and commercial components.
-#
-# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
-# your code to Maciej Mensfeld.
+# This code is part of Karafka Pro, a commercial component not licensed under LGPL.
+# See LICENSE for details.
 
 module Karafka
   module Pro
@@ -52,7 +44,7 @@ module Karafka
                   elsif !revoked?
                     # no need to check for manual seek because AJ consumer is internal and
                     # fully controlled by us
-                    seek(coordinator.seek_offset, false)
+                    seek(seek_offset, false)
                     resume
                   else
                     resume

data/lib/karafka/pro/processing/strategies/aj/dlq_ftr_mom.rb

@@ -1,15 +1,7 @@
 # frozen_string_literal: true
 
-# This Karafka component is a Pro component under a commercial license.
-# This Karafka component is NOT licensed under LGPL.
-#
-# All of the commercial components are present in the lib/karafka/pro directory of this
-# repository and their usage requires commercial license agreement.
-#
-# Karafka has also commercial-friendly license, commercial support and commercial components.
-#
-# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
-# your code to Maciej Mensfeld.
+# This code is part of Karafka Pro, a commercial component not licensed under LGPL.
+# See LICENSE for details.
 
 module Karafka
   module Pro

data/lib/karafka/pro/processing/strategies/aj/dlq_ftr_mom_vp.rb

@@ -1,15 +1,7 @@
 # frozen_string_literal: true
 
-# This Karafka component is a Pro component under a commercial license.
-# This Karafka component is NOT licensed under LGPL.
-#
-# All of the commercial components are present in the lib/karafka/pro directory of this
-# repository and their usage requires commercial license agreement.
-#
-# Karafka has also commercial-friendly license, commercial support and commercial components.
-#
-# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
-# your code to Maciej Mensfeld.
+# This code is part of Karafka Pro, a commercial component not licensed under LGPL.
+# See LICENSE for details.
 
 module Karafka
   module Pro

data/lib/karafka/pro/processing/strategies/aj/dlq_lrj_mom.rb

@@ -1,15 +1,7 @@
 # frozen_string_literal: true
 
-# This Karafka component is a Pro component under a commercial license.
-# This Karafka component is NOT licensed under LGPL.
-#
-# All of the commercial components are present in the lib/karafka/pro directory of this
-# repository and their usage requires commercial license agreement.
-#
-# Karafka has also commercial-friendly license, commercial support and commercial components.
-#
-# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
-# your code to Maciej Mensfeld.
+# This code is part of Karafka Pro, a commercial component not licensed under LGPL.
+# See LICENSE for details.
 
 module Karafka
   module Pro
@@ -44,7 +36,7 @@ module Karafka
 
                   # no need to check for manual seek because AJ consumer is internal and
                   # fully controlled by us
-                  seek(coordinator.seek_offset, false) unless revoked?
+                  seek(seek_offset, false) unless revoked?
 
                   resume
                 else

data/lib/karafka/pro/processing/strategies/aj/dlq_lrj_mom_vp.rb

@@ -1,15 +1,7 @@
 # frozen_string_literal: true
 
-# This Karafka component is a Pro component under a commercial license.
-# This Karafka component is NOT licensed under LGPL.
-#
-# All of the commercial components are present in the lib/karafka/pro directory of this
-# repository and their usage requires commercial license agreement.
-#
-# Karafka has also commercial-friendly license, commercial support and commercial components.
-#
-# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
-# your code to Maciej Mensfeld.
+# This code is part of Karafka Pro, a commercial component not licensed under LGPL.
+# See LICENSE for details.
 
 module Karafka
   module Pro
@@ -48,7 +40,7 @@ module Karafka
                   mark_as_consumed(last_group_message) unless revoked?
                   # no need to check for manual seek because AJ consumer is internal and
                   # fully controlled by us
-                  seek(coordinator.seek_offset, false) unless revoked?
+                  seek(seek_offset, false) unless revoked?
 
                   resume
                 else

data/lib/karafka/pro/processing/strategies/aj/dlq_mom.rb

@@ -1,15 +1,7 @@
 # frozen_string_literal: true
 
-# This Karafka component is a Pro component under a commercial license.
-# This Karafka component is NOT licensed under LGPL.
-#
-# All of the commercial components are present in the lib/karafka/pro directory of this
-# repository and their usage requires commercial license agreement.
-#
-# Karafka has also commercial-friendly license, commercial support and commercial components.
-#
-# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
-# your code to Maciej Mensfeld.
+# This code is part of Karafka Pro, a commercial component not licensed under LGPL.
+# See LICENSE for details.
 
 module Karafka
   module Pro

data/lib/karafka/pro/processing/strategies/aj/dlq_mom_vp.rb

@@ -1,15 +1,7 @@
 # frozen_string_literal: true
 
-# This Karafka component is a Pro component under a commercial license.
-# This Karafka component is NOT licensed under LGPL.
-#
-# All of the commercial components are present in the lib/karafka/pro directory of this
-# repository and their usage requires commercial license agreement.
-#
-# Karafka has also commercial-friendly license, commercial support and commercial components.
-#
-# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
-# your code to Maciej Mensfeld.
+# This code is part of Karafka Pro, a commercial component not licensed under LGPL.
+# See LICENSE for details.
 
 module Karafka
   module Pro

data/lib/karafka/pro/processing/strategies/aj/ftr_lrj_mom.rb

@@ -1,15 +1,7 @@
 # frozen_string_literal: true
 
-# This Karafka component is a Pro component under a commercial license.
-# This Karafka component is NOT licensed under LGPL.
-#
-# All of the commercial components are present in the lib/karafka/pro directory of this
-# repository and their usage requires commercial license agreement.
-#
-# Karafka has also commercial-friendly license, commercial support and commercial components.
-#
-# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
-# your code to Maciej Mensfeld.
+# This code is part of Karafka Pro, a commercial component not licensed under LGPL.
+# See LICENSE for details.
 
 module Karafka
   module Pro

data/lib/karafka/pro/processing/strategies/aj/ftr_lrj_mom_vp.rb

@@ -1,15 +1,7 @@
 # frozen_string_literal: true
 
-# This Karafka component is a Pro component under a commercial license.
-# This Karafka component is NOT licensed under LGPL.
-#
-# All of the commercial components are present in the lib/karafka/pro directory of this
-# repository and their usage requires commercial license agreement.
-#
-# Karafka has also commercial-friendly license, commercial support and commercial components.
-#
-# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
-# your code to Maciej Mensfeld.
+# This code is part of Karafka Pro, a commercial component not licensed under LGPL.
+# See LICENSE for details.
 
 module Karafka
   module Pro
@@ -48,7 +40,7 @@ module Karafka
                   elsif !revoked?
                     # no need to check for manual seek because AJ consumer is internal and
                     # fully controlled by us
-                    seek(coordinator.seek_offset, false)
+                    seek(seek_offset, false)
                     resume
                   else
                     resume

data/lib/karafka/pro/processing/strategies/aj/ftr_mom.rb

@@ -1,15 +1,7 @@
 # frozen_string_literal: true
 
-# This Karafka component is a Pro component under a commercial license.
-# This Karafka component is NOT licensed under LGPL.
-#
-# All of the commercial components are present in the lib/karafka/pro directory of this
-# repository and their usage requires commercial license agreement.
-#
-# Karafka has also commercial-friendly license, commercial support and commercial components.
-#
-# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
-# your code to Maciej Mensfeld.
+# This code is part of Karafka Pro, a commercial component not licensed under LGPL.
+# See LICENSE for details.
 
 module Karafka
   module Pro

data/lib/karafka/pro/processing/strategies/aj/ftr_mom_vp.rb

@@ -1,15 +1,7 @@
 # frozen_string_literal: true
 
-# This Karafka component is a Pro component under a commercial license.
-# This Karafka component is NOT licensed under LGPL.
-#
-# All of the commercial components are present in the lib/karafka/pro directory of this
-# repository and their usage requires commercial license agreement.
-#
-# Karafka has also commercial-friendly license, commercial support and commercial components.
-#
-# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
-# your code to Maciej Mensfeld.
+# This code is part of Karafka Pro, a commercial component not licensed under LGPL.
+# See LICENSE for details.
 
 module Karafka
   module Pro

data/lib/karafka/pro/processing/strategies/aj/lrj_mom.rb

@@ -1,15 +1,7 @@
 # frozen_string_literal: true
 
-# This Karafka component is a Pro component under a commercial license.
-# This Karafka component is NOT licensed under LGPL.
-#
-# All of the commercial components are present in the lib/karafka/pro directory of this
-# repository and their usage requires commercial license agreement.
-#
-# Karafka has also commercial-friendly license, commercial support and commercial components.
-#
-# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
-# your code to Maciej Mensfeld.
+# This code is part of Karafka Pro, a commercial component not licensed under LGPL.
+# See LICENSE for details.
 
 module Karafka
   module Pro

data/lib/karafka/pro/processing/strategies/aj/lrj_mom_vp.rb

@@ -1,15 +1,7 @@
 # frozen_string_literal: true
 
-# This Karafka component is a Pro component under a commercial license.
-# This Karafka component is NOT licensed under LGPL.
-#
-# All of the commercial components are present in the lib/karafka/pro directory of this
-# repository and their usage requires commercial license agreement.
-#
-# Karafka has also commercial-friendly license, commercial support and commercial components.
-#
-# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
-# your code to Maciej Mensfeld.
+# This code is part of Karafka Pro, a commercial component not licensed under LGPL.
+# See LICENSE for details.
 
 module Karafka
   module Pro
@@ -51,7 +43,7 @@ module Karafka
 
                   # no need to check for manual seek because AJ consumer is internal and
                   # fully controlled by us
-                  seek(coordinator.seek_offset, false) unless revoked?
+                  seek(seek_offset, false) unless revoked?
 
                   resume
                 else

data/lib/karafka/pro/processing/strategies/aj/mom.rb

@@ -1,15 +1,7 @@
 # frozen_string_literal: true
 
-# This Karafka component is a Pro component under a commercial license.
-# This Karafka component is NOT licensed under LGPL.
-#
-# All of the commercial components are present in the lib/karafka/pro directory of this
-# repository and their usage requires commercial license agreement.
-#
-# Karafka has also commercial-friendly license, commercial support and commercial components.
-#
-# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
-# your code to Maciej Mensfeld.
+# This code is part of Karafka Pro, a commercial component not licensed under LGPL.
+# See LICENSE for details.
 
 module Karafka
   module Pro

data/lib/karafka/pro/processing/strategies/aj/mom_vp.rb

@@ -1,15 +1,7 @@
 # frozen_string_literal: true
 
-# This Karafka component is a Pro component under a commercial license.
-# This Karafka component is NOT licensed under LGPL.
-#
-# All of the commercial components are present in the lib/karafka/pro directory of this
-# repository and their usage requires commercial license agreement.
-#
-# Karafka has also commercial-friendly license, commercial support and commercial components.
-#
-# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
-# your code to Maciej Mensfeld.
+# This code is part of Karafka Pro, a commercial component not licensed under LGPL.
+# See LICENSE for details.
 
 module Karafka
   module Pro

data/lib/karafka/pro/processing/strategies/base.rb

@@ -1,15 +1,7 @@
 # frozen_string_literal: true
 
-# This Karafka component is a Pro component under a commercial license.
-# This Karafka component is NOT licensed under LGPL.
-#
-# All of the commercial components are present in the lib/karafka/pro directory of this
-# repository and their usage requires commercial license agreement.
-#
-# Karafka has also commercial-friendly license, commercial support and commercial components.
-#
-# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
-# your code to Maciej Mensfeld.
+# This code is part of Karafka Pro, a commercial component not licensed under LGPL.
+# See LICENSE for details.
 
 module Karafka
   module Pro

data/lib/karafka/pro/processing/strategies/default.rb

@@ -1,15 +1,7 @@
 # frozen_string_literal: true
 
-# This Karafka component is a Pro component under a commercial license.
-# This Karafka component is NOT licensed under LGPL.
-#
-# All of the commercial components are present in the lib/karafka/pro directory of this
-# repository and their usage requires commercial license agreement.
-#
-# Karafka has also commercial-friendly license, commercial support and commercial components.
-#
-# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
-# your code to Maciej Mensfeld.
+# This code is part of Karafka Pro, a commercial component not licensed under LGPL.
+# See LICENSE for details.
 
 module Karafka
   module Pro
@@ -62,9 +54,9 @@ module Karafka
             else
               # seek offset can be nil only in case `#seek` was invoked with offset reset request
               # In case like this we ignore marking
-              return true if coordinator.seek_offset.nil?
+              return true if seek_offset.nil?
               # Ignore earlier offsets than the one we already committed
-              return true if coordinator.seek_offset > message.offset
+              return true if seek_offset > message.offset
               return false if revoked?
 
               # If we are not inside a transaction but this is a transactional topic, we mark with
@@ -77,7 +69,7 @@ module Karafka
 
               return revoked? unless stored
 
-              coordinator.seek_offset = message.offset + 1
+              self.seek_offset = message.offset + 1
             end
 
             true
@@ -99,9 +91,9 @@ module Karafka
             else
               # seek offset can be nil only in case `#seek` was invoked with offset reset request
               # In case like this we ignore marking
-              return true if coordinator.seek_offset.nil?
+              return true if seek_offset.nil?
               # Ignore earlier offsets than the one we already committed
-              return true if coordinator.seek_offset > message.offset
+              return true if seek_offset > message.offset
               return false if revoked?
 
               # If we are not inside a transaction but this is a transactional topic, we mark with
@@ -114,7 +106,7 @@ module Karafka
 
               return revoked? unless stored
 
-              coordinator.seek_offset = message.offset + 1
+              self.seek_offset = message.offset + 1
             end
 
             true
@@ -143,52 +135,57 @@ module Karafka
           #   way, so the message producing aliases operate from within transactions and since the
           #   producer in transaction is locked, it will prevent other threads from using it.
           def transaction(active_producer = producer)
-            default_producer = producer
-            self.producer = active_producer
+            default_producer = nil
+            transaction_started = nil
 
-            transaction_started = false
+            monitor.instrument('consumer.consuming.transaction', caller: self) do
+              default_producer = producer
+              self.producer = active_producer
 
-            # Prevent from nested transactions. It would not make any sense
-            raise Errors::TransactionAlreadyInitializedError if @_in_transaction
+              transaction_started = false
 
-            transaction_started = true
-            @_transaction_marked = []
-            @_in_transaction = true
-            @_in_transaction_marked = false
+              # Prevent from nested transactions. It would not make any sense
+              raise Errors::TransactionAlreadyInitializedError if @_in_transaction
 
-            producer.transaction do
-              yield
+              transaction_started = true
+              @_transaction_marked = []
+              @_in_transaction = true
+              @_in_transaction_marked = false
 
-              # Ensure this transaction is rolled back if we have lost the ownership of this
-              # transaction. We do it only for transactions that contain offset management as for
-              # producer only, this is not relevant.
-              raise Errors::AssignmentLostError if @_in_transaction_marked && revoked?
-            end
+              producer.transaction do
+                yield
 
-            @_in_transaction = false
-
-            # This offset is already stored in transaction but we set it here anyhow because we
-            # want to make sure our internal in-memory state is aligned with the transaction
-            #
-            # @note We never need to use the blocking `#mark_as_consumed!` here because the offset
-            #   anyhow was already stored during the transaction
-            #
-            # @note Since the offset could have been already stored in Kafka (could have because
-            #   you can have transactions without marking), we use the `@_in_transaction_marked`
-            #   state to decide if we need to dispatch the offset via client at all
-            #   (if post transaction, then we do not have to)
-            #
-            # @note In theory we could only keep reference to the most recent marking and reject
-            #   others. We however do not do it for two reasons:
-            #   - User may have non standard flow relying on some alternative order and we want to
-            #     mimic this
-            #   - Complex strategies like VPs can use this in VPs to mark in parallel without
-            #     having to redefine the transactional flow completely
-            @_transaction_marked.each do |marking|
-              marking.pop ? mark_as_consumed(*marking) : mark_as_consumed!(*marking)
-            end
+                # Ensure this transaction is rolled back if we have lost the ownership of this
+                # transaction. We do it only for transactions that contain offset management as for
+                # producer only, this is not relevant.
+                raise Errors::AssignmentLostError if @_in_transaction_marked && revoked?
+              end
 
-            true
+              @_in_transaction = false
+
+              # This offset is already stored in transaction but we set it here anyhow because we
+              # want to make sure our internal in-memory state is aligned with the transaction
+              #
+              # @note We never need to use the blocking `#mark_as_consumed!` here because the
+              #   offset anyhow was already stored during the transaction
+              #
+              # @note Since the offset could have been already stored in Kafka (could have because
+              #   you can have transactions without marking), we use the `@_in_transaction_marked`
+              #   state to decide if we need to dispatch the offset via client at all
+              #   (if post transaction, then we do not have to)
+              #
+              # @note In theory we could only keep reference to the most recent marking and reject
+              #   others. We however do not do it for two reasons:
+              #   - User may have non standard flow relying on some alternative order and we want
+              #     to mimic this
+              #   - Complex strategies like VPs can use this in VPs to mark in parallel without
+              #     having to redefine the transactional flow completely
+              @_transaction_marked.each do |marking|
+                marking.pop ? mark_as_consumed(*marking) : mark_as_consumed!(*marking)
+              end
+
+              true
+            end
           ensure
             self.producer = default_producer
 
@@ -254,15 +251,15 @@ module Karafka
           def mark_in_memory(message)
             # seek offset can be nil only in case `#seek` was invoked with offset reset request
             # In case like this we ignore marking
-            return true if coordinator.seek_offset.nil?
+            return true if seek_offset.nil?
             # Ignore earlier offsets than the one we already committed
-            return true if coordinator.seek_offset > message.offset
+            return true if seek_offset > message.offset
             return false if revoked?
 
             # If we have already marked this successfully in a transaction that was running
             # we should not mark it again with the client offset delegation but instead we should
             # just align the in-memory state
-            coordinator.seek_offset = message.offset + 1
+            self.seek_offset = message.offset + 1
 
             true
           end

data/lib/karafka/pro/processing/strategies/dlq/default.rb

@@ -1,15 +1,7 @@
 # frozen_string_literal: true
 
-# This Karafka component is a Pro component under a commercial license.
-# This Karafka component is NOT licensed under LGPL.
-#
-# All of the commercial components are present in the lib/karafka/pro directory of this
-# repository and their usage requires commercial license agreement.
-#
-# Karafka has also commercial-friendly license, commercial support and commercial components.
-#
-# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
-# your code to Maciej Mensfeld.
+# This code is part of Karafka Pro, a commercial component not licensed under LGPL.
+# See LICENSE for details.
 
 module Karafka
   module Pro
@@ -90,7 +82,7 @@ module Karafka
             #   information if this message was from marked offset or figured out via mom flow
             def find_skippable_message
               skippable_message = messages.find do |msg|
-                coordinator.marked? && msg.offset == coordinator.seek_offset
+                coordinator.marked? && msg.offset == seek_offset
               end
 
               # If we don't have the message matching the last comitted offset, it means that
@@ -139,7 +131,7 @@ module Karafka
                 if mark_after_dispatch?
                   mark_dispatched_to_dlq(skippable_message)
                 else
-                  coordinator.seek_offset = skippable_message.offset + 1
+                  self.seek_offset = skippable_message.offset + 1
                 end
               end
 
@@ -234,7 +226,7 @@ module Karafka
               coordinator.pause_tracker.reset
 
               # Always backoff after DLQ dispatch even on skip to prevent overloads on errors
-              pause(coordinator.seek_offset, nil, false)
+              pause(seek_offset, nil, false)
             end
 
             # Marks message that went to DLQ (if applicable) based on the requested method

data/lib/karafka/pro/processing/strategies/dlq/ftr.rb

@@ -1,15 +1,7 @@
 # frozen_string_literal: true
 
-# This Karafka component is a Pro component under a commercial license.
-# This Karafka component is NOT licensed under LGPL.
-#
-# All of the commercial components are present in the lib/karafka/pro directory of this
-# repository and their usage requires commercial license agreement.
-#
-# Karafka has also commercial-friendly license, commercial support and commercial components.
-#
-# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
-# your code to Maciej Mensfeld.
+# This code is part of Karafka Pro, a commercial component not licensed under LGPL.
+# See LICENSE for details.
 
 module Karafka
   module Pro

data/lib/karafka/pro/processing/strategies/dlq/ftr_lrj.rb

@@ -1,15 +1,7 @@
 # frozen_string_literal: true
 
-# This Karafka component is a Pro component under a commercial license.
-# This Karafka component is NOT licensed under LGPL.
-#
-# All of the commercial components are present in the lib/karafka/pro directory of this
-# repository and their usage requires commercial license agreement.
-#
-# Karafka has also commercial-friendly license, commercial support and commercial components.
-#
-# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
-# your code to Maciej Mensfeld.
+# This code is part of Karafka Pro, a commercial component not licensed under LGPL.
+# See LICENSE for details.
 
 module Karafka
   module Pro
@@ -48,7 +40,7 @@ module Karafka
                   if coordinator.filtered? && !revoked?
                     handle_post_filtering
                   elsif !revoked? && !coordinator.manual_seek?
-                    seek(coordinator.seek_offset, false)
+                    seek(seek_offset, false)
                     resume
                   else
                     resume

data/lib/karafka/pro/processing/strategies/dlq/ftr_lrj_mom.rb

@@ -1,15 +1,7 @@
 # frozen_string_literal: true
 
-# This Karafka component is a Pro component under a commercial license.
-# This Karafka component is NOT licensed under LGPL.
-#
-# All of the commercial components are present in the lib/karafka/pro directory of this
-# repository and their usage requires commercial license agreement.
-#
-# Karafka has also commercial-friendly license, commercial support and commercial components.
-#
-# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
-# your code to Maciej Mensfeld.
+# This code is part of Karafka Pro, a commercial component not licensed under LGPL.
+# See LICENSE for details.
 
 module Karafka
   module Pro
@@ -58,7 +50,7 @@ module Karafka
                     if mark_after_dispatch?
                       mark_dispatched_to_dlq(skippable_message)
                     else
-                      coordinator.seek_offset = skippable_message.offset + 1
+                      self.seek_offset = skippable_message.offset + 1
                     end
                   end
                 end