karafka 2.4.15 → 2.4.17

data/lib/karafka/pro/processing/strategies/aj/ftr_mom.rb

@@ -1,15 +1,7 @@
 # frozen_string_literal: true
 
-# This Karafka component is a Pro component under a commercial license.
-# This Karafka component is NOT licensed under LGPL.
-#
-# All of the commercial components are present in the lib/karafka/pro directory of this
-# repository and their usage requires commercial license agreement.
-#
-# Karafka has also commercial-friendly license, commercial support and commercial components.
-#
-# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
-# your code to Maciej Mensfeld.
+# This code is part of Karafka Pro, a commercial component not licensed under LGPL.
+# See LICENSE for details.
 
 module Karafka
   module Pro

data/lib/karafka/pro/processing/strategies/aj/ftr_mom_vp.rb

@@ -1,15 +1,7 @@
 # frozen_string_literal: true
 
-# This Karafka component is a Pro component under a commercial license.
-# This Karafka component is NOT licensed under LGPL.
-#
-# All of the commercial components are present in the lib/karafka/pro directory of this
-# repository and their usage requires commercial license agreement.
-#
-# Karafka has also commercial-friendly license, commercial support and commercial components.
-#
-# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
-# your code to Maciej Mensfeld.
+# This code is part of Karafka Pro, a commercial component not licensed under LGPL.
+# See LICENSE for details.
 
 module Karafka
   module Pro

data/lib/karafka/pro/processing/strategies/aj/lrj_mom.rb

@@ -1,15 +1,7 @@
 # frozen_string_literal: true
 
-# This Karafka component is a Pro component under a commercial license.
-# This Karafka component is NOT licensed under LGPL.
-#
-# All of the commercial components are present in the lib/karafka/pro directory of this
-# repository and their usage requires commercial license agreement.
-#
-# Karafka has also commercial-friendly license, commercial support and commercial components.
-#
-# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
-# your code to Maciej Mensfeld.
+# This code is part of Karafka Pro, a commercial component not licensed under LGPL.
+# See LICENSE for details.
 
 module Karafka
   module Pro

data/lib/karafka/pro/processing/strategies/aj/lrj_mom_vp.rb

@@ -1,15 +1,7 @@
 # frozen_string_literal: true
 
-# This Karafka component is a Pro component under a commercial license.
-# This Karafka component is NOT licensed under LGPL.
-#
-# All of the commercial components are present in the lib/karafka/pro directory of this
-# repository and their usage requires commercial license agreement.
-#
-# Karafka has also commercial-friendly license, commercial support and commercial components.
-#
-# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
-# your code to Maciej Mensfeld.
+# This code is part of Karafka Pro, a commercial component not licensed under LGPL.
+# See LICENSE for details.
 
 module Karafka
   module Pro
@@ -51,7 +43,7 @@ module Karafka
 
                   # no need to check for manual seek because AJ consumer is internal and
                   # fully controlled by us
-                  seek(coordinator.seek_offset, false) unless revoked?
+                  seek(seek_offset, false) unless revoked?
 
                   resume
                 else
@@ -69,8 +61,8 @@ module Karafka
                 coordinator.revoke
               end
 
-              Karafka.monitor.instrument('consumer.revoke', caller: self)
-              Karafka.monitor.instrument('consumer.revoked', caller: self) do
+              monitor.instrument('consumer.revoke', caller: self)
+              monitor.instrument('consumer.revoked', caller: self) do
                 revoked
               end
             ensure

data/lib/karafka/pro/processing/strategies/aj/mom.rb

@@ -1,15 +1,7 @@
 # frozen_string_literal: true
 
-# This Karafka component is a Pro component under a commercial license.
-# This Karafka component is NOT licensed under LGPL.
-#
-# All of the commercial components are present in the lib/karafka/pro directory of this
-# repository and their usage requires commercial license agreement.
-#
-# Karafka has also commercial-friendly license, commercial support and commercial components.
-#
-# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
-# your code to Maciej Mensfeld.
+# This code is part of Karafka Pro, a commercial component not licensed under LGPL.
+# See LICENSE for details.
 
 module Karafka
   module Pro

data/lib/karafka/pro/processing/strategies/aj/mom_vp.rb

@@ -1,15 +1,7 @@
 # frozen_string_literal: true
 
-# This Karafka component is a Pro component under a commercial license.
-# This Karafka component is NOT licensed under LGPL.
-#
-# All of the commercial components are present in the lib/karafka/pro directory of this
-# repository and their usage requires commercial license agreement.
-#
-# Karafka has also commercial-friendly license, commercial support and commercial components.
-#
-# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
-# your code to Maciej Mensfeld.
+# This code is part of Karafka Pro, a commercial component not licensed under LGPL.
+# See LICENSE for details.
 
 module Karafka
   module Pro

data/lib/karafka/pro/processing/strategies/base.rb

@@ -1,15 +1,7 @@
 # frozen_string_literal: true
 
-# This Karafka component is a Pro component under a commercial license.
-# This Karafka component is NOT licensed under LGPL.
-#
-# All of the commercial components are present in the lib/karafka/pro directory of this
-# repository and their usage requires commercial license agreement.
-#
-# Karafka has also commercial-friendly license, commercial support and commercial components.
-#
-# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
-# your code to Maciej Mensfeld.
+# This code is part of Karafka Pro, a commercial component not licensed under LGPL.
+# See LICENSE for details.
 
 module Karafka
   module Pro

data/lib/karafka/pro/processing/strategies/default.rb

@@ -1,15 +1,7 @@
 # frozen_string_literal: true
 
-# This Karafka component is a Pro component under a commercial license.
-# This Karafka component is NOT licensed under LGPL.
-#
-# All of the commercial components are present in the lib/karafka/pro directory of this
-# repository and their usage requires commercial license agreement.
-#
-# Karafka has also commercial-friendly license, commercial support and commercial components.
-#
-# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
-# your code to Maciej Mensfeld.
+# This code is part of Karafka Pro, a commercial component not licensed under LGPL.
+# See LICENSE for details.
 
 module Karafka
   module Pro
@@ -54,28 +46,30 @@ module Karafka
           #   already processed but rather at the next one. This applies to both sync and async
           #   versions of this method.
           def mark_as_consumed(message, offset_metadata = @_current_offset_metadata)
+            # If we are inside a transaction than we can just mark as consumed within it
             if @_in_transaction
               mark_in_transaction(message, offset_metadata, true)
+            elsif @_in_transaction_marked
+              mark_in_memory(message)
             else
               # seek offset can be nil only in case `#seek` was invoked with offset reset request
               # In case like this we ignore marking
-              return true if coordinator.seek_offset.nil?
+              return true if seek_offset.nil?
               # Ignore earlier offsets than the one we already committed
-              return true if coordinator.seek_offset > message.offset
+              return true if seek_offset > message.offset
               return false if revoked?
 
-              # If we have already marked this successfully in a transaction that was running
-              # we should not mark it again with the client offset delegation but instead we should
-              # just align the in-memory state
-              if @_in_transaction_marked
-                coordinator.seek_offset = message.offset + 1
+              # If we are not inside a transaction but this is a transactional topic, we mark with
+              # artificially created transaction
+              stored = if producer.transactional?
+                         mark_with_transaction(message, offset_metadata, true)
+                       else
+                         client.mark_as_consumed(message, offset_metadata)
+                       end
 
-                return true
-              end
-
-              return revoked? unless client.mark_as_consumed(message, offset_metadata)
+              return revoked? unless stored
 
-              coordinator.seek_offset = message.offset + 1
+              self.seek_offset = message.offset + 1
             end
 
             true
@@ -92,23 +86,27 @@ module Karafka
           def mark_as_consumed!(message, offset_metadata = @_current_offset_metadata)
             if @_in_transaction
               mark_in_transaction(message, offset_metadata, false)
+            elsif @_in_transaction_marked
+              mark_in_memory(message)
             else
               # seek offset can be nil only in case `#seek` was invoked with offset reset request
               # In case like this we ignore marking
-              return true if coordinator.seek_offset.nil?
+              return true if seek_offset.nil?
               # Ignore earlier offsets than the one we already committed
-              return true if coordinator.seek_offset > message.offset
+              return true if seek_offset > message.offset
               return false if revoked?
 
-              if @_in_transaction_marked
-                coordinator.seek_offset = message.offset + 1
-
-                return true
-              end
+              # If we are not inside a transaction but this is a transactional topic, we mark with
+              # artificially created transaction
+              stored = if producer.transactional?
+                         mark_with_transaction(message, offset_metadata, false)
+                       else
+                         client.mark_as_consumed!(message, offset_metadata)
+                       end
 
-              return revoked? unless client.mark_as_consumed!(message, offset_metadata)
+              return revoked? unless stored
 
-              coordinator.seek_offset = message.offset + 1
+              self.seek_offset = message.offset + 1
             end
 
             true
@@ -128,7 +126,7 @@ module Karafka
           #   managing multiple producers. If not provided, default producer taken from `#producer`
           #   will be used.
           #
-          # @param block [Proc] code that we want to run in a transaction
+          # @yield code that we want to run in a transaction
           #
           # @note Please note, that if you provide the producer, it will reassign the producer of
           #   the consumer for the transaction time. This means, that in case you would even
@@ -136,43 +134,57 @@ module Karafka
           #   reassigned producer and not the initially used/assigned producer. It is done that
           #   way, so the message producing aliases operate from within transactions and since the
           #   producer in transaction is locked, it will prevent other threads from using it.
-          def transaction(active_producer = producer, &block)
-            default_producer = producer
-            self.producer = active_producer
-
-            transaction_started = false
-
-            # Prevent from nested transactions. It would not make any sense
-            raise Errors::TransactionAlreadyInitializedError if @_in_transaction
-
-            transaction_started = true
-            @_transaction_marked = []
-            @_in_transaction = true
-            @_in_transaction_marked = false
-
-            producer.transaction(&block)
-
-            @_in_transaction = false
-
-            # This offset is already stored in transaction but we set it here anyhow because we
-            # want to make sure our internal in-memory state is aligned with the transaction
-            #
-            # @note We never need to use the blocking `#mark_as_consumed!` here because the offset
-            #   anyhow was already stored during the transaction
-            #
-            # @note Since the offset could have been already stored in Kafka (could have because
-            #   you can have transactions without marking), we use the `@_in_transaction_marked`
-            #   state to decide if we need to dispatch the offset via client at all
-            #   (if post transaction, then we do not have to)
-            #
-            # @note In theory we could only keep reference to the most recent marking and reject
-            #   others. We however do not do it for two reasons:
-            #   - User may have non standard flow relying on some alternative order and we want to
-            #     mimic this
-            #   - Complex strategies like VPs can use this in VPs to mark in parallel without
-            #     having to redefine the transactional flow completely
-            @_transaction_marked.each do |marking|
-              marking.pop ? mark_as_consumed(*marking) : mark_as_consumed!(*marking)
+          def transaction(active_producer = producer)
+            default_producer = nil
+            transaction_started = nil
+
+            monitor.instrument('consumer.consuming.transaction', caller: self) do
+              default_producer = producer
+              self.producer = active_producer
+
+              transaction_started = false
+
+              # Prevent from nested transactions. It would not make any sense
+              raise Errors::TransactionAlreadyInitializedError if @_in_transaction
+
+              transaction_started = true
+              @_transaction_marked = []
+              @_in_transaction = true
+              @_in_transaction_marked = false
+
+              producer.transaction do
+                yield
+
+                # Ensure this transaction is rolled back if we have lost the ownership of this
+                # transaction. We do it only for transactions that contain offset management as for
+                # producer only, this is not relevant.
+                raise Errors::AssignmentLostError if @_in_transaction_marked && revoked?
+              end
+
+              @_in_transaction = false
+
+              # This offset is already stored in transaction but we set it here anyhow because we
+              # want to make sure our internal in-memory state is aligned with the transaction
+              #
+              # @note We never need to use the blocking `#mark_as_consumed!` here because the
+              #   offset anyhow was already stored during the transaction
+              #
+              # @note Since the offset could have been already stored in Kafka (could have because
+              #   you can have transactions without marking), we use the `@_in_transaction_marked`
+              #   state to decide if we need to dispatch the offset via client at all
+              #   (if post transaction, then we do not have to)
+              #
+              # @note In theory we could only keep reference to the most recent marking and reject
+              #   others. We however do not do it for two reasons:
+              #   - User may have non standard flow relying on some alternative order and we want
+              #     to mimic this
+              #   - Complex strategies like VPs can use this in VPs to mark in parallel without
+              #     having to redefine the transactional flow completely
+              @_transaction_marked.each do |marking|
+                marking.pop ? mark_as_consumed(*marking) : mark_as_consumed!(*marking)
+              end
+
+              true
             end
           ensure
             self.producer = default_producer
@@ -206,9 +218,55 @@ module Karafka
             @_transaction_marked << [message, offset_metadata, async]
           end
 
+          # @private
+          # @param message [Messages::Message] message we want to commit inside of a transaction
+          # @param offset_metadata [String, nil] offset metadata or nil if none
+          # @param async [Boolean] should we mark in async or sync way (applicable only to post
+          #   transaction state synchronization usage as within transaction it is always sync)
+          # @return [Boolean] false if marking failed otherwise true
+          def mark_with_transaction(message, offset_metadata, async)
+            # This flag is used by VPs to differentiate between user initiated transactions and
+            # post-execution system transactions.
+            @_transaction_internal = true
+
+            transaction do
+              mark_in_transaction(message, offset_metadata, async)
+            end
+
+            true
+          # We handle both cases here because this is a private API for internal usage and we want
+          # the post-user code execution marking with transactional producer to result in a
+          # boolean state of marking for further framework flow. This is a normalization to make it
+          # behave the same way as it would behave with a non-transactional one
+          rescue ::Rdkafka::RdkafkaError, Errors::AssignmentLostError
+            false
+          ensure
+            @_transaction_internal = false
+          end
+
+          # Marks the current state only in memory as the offset marking has already happened
+          # using the producer transaction
+          # @param message [Messages::Message] last successfully processed message.
+          # @return [Boolean] true if all good, false if we lost assignment and no point in marking
+          def mark_in_memory(message)
+            # seek offset can be nil only in case `#seek` was invoked with offset reset request
+            # In case like this we ignore marking
+            return true if seek_offset.nil?
+            # Ignore earlier offsets than the one we already committed
+            return true if seek_offset > message.offset
+            return false if revoked?
+
+            # If we have already marked this successfully in a transaction that was running
+            # we should not mark it again with the client offset delegation but instead we should
+            # just align the in-memory state
+            self.seek_offset = message.offset + 1
+
+            true
+          end
+
           # No actions needed for the standard flow here
           def handle_before_schedule_consume
-            Karafka.monitor.instrument('consumer.before_schedule_consume', caller: self)
+            monitor.instrument('consumer.before_schedule_consume', caller: self)
 
             nil
           end
@@ -227,8 +285,8 @@ module Karafka
             # This can happen primarily when an LRJ job gets to the internal worker queue and
             # this partition is revoked prior processing.
             unless revoked?
-              Karafka.monitor.instrument('consumer.consume', caller: self)
-              Karafka.monitor.instrument('consumer.consumed', caller: self) do
+              monitor.instrument('consumer.consume', caller: self)
+              monitor.instrument('consumer.consumed', caller: self) do
                 consume
               end
             end
@@ -274,8 +332,8 @@ module Karafka
               coordinator.revoke
             end
 
-            Karafka.monitor.instrument('consumer.revoke', caller: self)
-            Karafka.monitor.instrument('consumer.revoked', caller: self) do
+            monitor.instrument('consumer.revoke', caller: self)
+            monitor.instrument('consumer.revoked', caller: self) do
               revoked
             end
           ensure
@@ -284,15 +342,15 @@ module Karafka
 
           # No action needed for the tick standard flow
           def handle_before_schedule_tick
-            Karafka.monitor.instrument('consumer.before_schedule_tick', caller: self)
+            monitor.instrument('consumer.before_schedule_tick', caller: self)
 
             nil
           end
 
           # Runs the consumer `#tick` method with reporting
           def handle_tick
-            Karafka.monitor.instrument('consumer.tick', caller: self)
-            Karafka.monitor.instrument('consumer.ticked', caller: self) do
+            monitor.instrument('consumer.tick', caller: self)
+            monitor.instrument('consumer.ticked', caller: self) do
               tick
             end
           ensure

data/lib/karafka/pro/processing/strategies/dlq/default.rb

@@ -1,15 +1,7 @@
 # frozen_string_literal: true
 
-# This Karafka component is a Pro component under a commercial license.
-# This Karafka component is NOT licensed under LGPL.
-#
-# All of the commercial components are present in the lib/karafka/pro directory of this
-# repository and their usage requires commercial license agreement.
-#
-# Karafka has also commercial-friendly license, commercial support and commercial components.
-#
-# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
-# your code to Maciej Mensfeld.
+# This code is part of Karafka Pro, a commercial component not licensed under LGPL.
+# See LICENSE for details.
 
 module Karafka
   module Pro
@@ -90,7 +82,7 @@ module Karafka
             #   information if this message was from marked offset or figured out via mom flow
             def find_skippable_message
               skippable_message = messages.find do |msg|
-                coordinator.marked? && msg.offset == coordinator.seek_offset
+                coordinator.marked? && msg.offset == seek_offset
               end
 
               # If we don't have the message matching the last comitted offset, it means that
@@ -119,7 +111,7 @@ module Karafka
               )
 
               # Notify about dispatch on the events bus
-              Karafka.monitor.instrument(
+              monitor.instrument(
                 'dead_letter_queue.dispatched',
                 caller: self,
                 message: skippable_message
@@ -139,7 +131,7 @@ module Karafka
                 if mark_after_dispatch?
                   mark_dispatched_to_dlq(skippable_message)
                 else
-                  coordinator.seek_offset = skippable_message.offset + 1
+                  self.seek_offset = skippable_message.offset + 1
                 end
               end
 
@@ -234,7 +226,7 @@ module Karafka
               coordinator.pause_tracker.reset
 
               # Always backoff after DLQ dispatch even on skip to prevent overloads on errors
-              pause(coordinator.seek_offset, nil, false)
+              pause(seek_offset, nil, false)
             end
 
             # Marks message that went to DLQ (if applicable) based on the requested method

data/lib/karafka/pro/processing/strategies/dlq/ftr.rb

@@ -1,15 +1,7 @@
 # frozen_string_literal: true
 
-# This Karafka component is a Pro component under a commercial license.
-# This Karafka component is NOT licensed under LGPL.
-#
-# All of the commercial components are present in the lib/karafka/pro directory of this
-# repository and their usage requires commercial license agreement.
-#
-# Karafka has also commercial-friendly license, commercial support and commercial components.
-#
-# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
-# your code to Maciej Mensfeld.
+# This code is part of Karafka Pro, a commercial component not licensed under LGPL.
+# See LICENSE for details.
 
 module Karafka
   module Pro

data/lib/karafka/pro/processing/strategies/dlq/ftr_lrj.rb

@@ -1,15 +1,7 @@
 # frozen_string_literal: true
 
-# This Karafka component is a Pro component under a commercial license.
-# This Karafka component is NOT licensed under LGPL.
-#
-# All of the commercial components are present in the lib/karafka/pro directory of this
-# repository and their usage requires commercial license agreement.
-#
-# Karafka has also commercial-friendly license, commercial support and commercial components.
-#
-# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
-# your code to Maciej Mensfeld.
+# This code is part of Karafka Pro, a commercial component not licensed under LGPL.
+# See LICENSE for details.
 
 module Karafka
   module Pro
@@ -48,7 +40,7 @@ module Karafka
                   if coordinator.filtered? && !revoked?
                     handle_post_filtering
                   elsif !revoked? && !coordinator.manual_seek?
-                    seek(coordinator.seek_offset, false)
+                    seek(seek_offset, false)
                     resume
                   else
                     resume

data/lib/karafka/pro/processing/strategies/dlq/ftr_lrj_mom.rb

@@ -1,15 +1,7 @@
 # frozen_string_literal: true
 
-# This Karafka component is a Pro component under a commercial license.
-# This Karafka component is NOT licensed under LGPL.
-#
-# All of the commercial components are present in the lib/karafka/pro directory of this
-# repository and their usage requires commercial license agreement.
-#
-# Karafka has also commercial-friendly license, commercial support and commercial components.
-#
-# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
-# your code to Maciej Mensfeld.
+# This code is part of Karafka Pro, a commercial component not licensed under LGPL.
+# See LICENSE for details.
 
 module Karafka
   module Pro
@@ -58,7 +50,7 @@ module Karafka
                     if mark_after_dispatch?
                       mark_dispatched_to_dlq(skippable_message)
                     else
-                      coordinator.seek_offset = skippable_message.offset + 1
+                      self.seek_offset = skippable_message.offset + 1
                     end
                   end
                 end

data/lib/karafka/pro/processing/strategies/dlq/ftr_lrj_mom_vp.rb

@@ -1,15 +1,7 @@
 # frozen_string_literal: true
 
-# This Karafka component is a Pro component under a commercial license.
-# This Karafka component is NOT licensed under LGPL.
-#
-# All of the commercial components are present in the lib/karafka/pro directory of this
-# repository and their usage requires commercial license agreement.
-#
-# Karafka has also commercial-friendly license, commercial support and commercial components.
-#
-# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
-# your code to Maciej Mensfeld.
+# This code is part of Karafka Pro, a commercial component not licensed under LGPL.
+# See LICENSE for details.
 
 module Karafka
   module Pro

data/lib/karafka/pro/processing/strategies/dlq/ftr_lrj_vp.rb

@@ -1,15 +1,7 @@
 # frozen_string_literal: true
 
-# This Karafka component is a Pro component under a commercial license.
-# This Karafka component is NOT licensed under LGPL.
-#
-# All of the commercial components are present in the lib/karafka/pro directory of this
-# repository and their usage requires commercial license agreement.
-#
-# Karafka has also commercial-friendly license, commercial support and commercial components.
-#
-# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
-# your code to Maciej Mensfeld.
+# This code is part of Karafka Pro, a commercial component not licensed under LGPL.
+# See LICENSE for details.
 
 module Karafka
   module Pro

data/lib/karafka/pro/processing/strategies/dlq/ftr_mom.rb

@@ -1,15 +1,7 @@
 # frozen_string_literal: true
 
-# This Karafka component is a Pro component under a commercial license.
-# This Karafka component is NOT licensed under LGPL.
-#
-# All of the commercial components are present in the lib/karafka/pro directory of this
-# repository and their usage requires commercial license agreement.
-#
-# Karafka has also commercial-friendly license, commercial support and commercial components.
-#
-# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
-# your code to Maciej Mensfeld.
+# This code is part of Karafka Pro, a commercial component not licensed under LGPL.
+# See LICENSE for details.
 
 module Karafka
   module Pro
@@ -49,7 +41,7 @@ module Karafka
                     if mark_after_dispatch?
                       mark_dispatched_to_dlq(skippable_message)
                     else
-                      coordinator.seek_offset = skippable_message.offset + 1
+                      self.seek_offset = skippable_message.offset + 1
                     end
                   end
                 end